Skip to content

Commit 4caabcc

Browse files
lnotspotllnotspotl
authored
Merge pull request #1418 from luxonis/lnotspotl/combined_python_wheels_opt
Bundle related python wheels together
2 parents de42cf6 + 5c44005 commit 4caabcc

File tree

8 files changed

+641
-128
lines changed

8 files changed

+641
-128
lines changed

‎.github/workflows/python-main.yml‎

Lines changed: 112 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -239,22 +239,28 @@ jobs:
239239
run: |
240240
python -m pip install --upgrade pip
241241
- name: Building wheels
242-
run: cd bindings/python && python -m pip wheel . -w ./wheelhouse/audited/ --verbose
242+
run: cd bindings/python && python -m pip wheel . -w ./wheelhouse/ --verbose
243243
- name: Print out vcpkg logs if building port fails
244244
if: failure() # Only run this if the build step fails
245245
run: cd bindings/python && bash ./ci/show_vcpkg_logs.sh
246+
- name: Install delvewheel
247+
run: python -m pip install delvewheel
248+
- name: List wheelhouse
249+
run: cd bindings/python && ls ./wheelhouse
250+
- name: Auditing wheels
251+
shell: bash
252+
run: |
253+
cd bindings/python
254+
EXTRA_DLL_PATH=$(find ./build -type d -name "temp.win*" -exec ls -d {} \; | head -n 1)/Release/Release
255+
echo "delvewheel extra dll path: $EXTRA_DLL_PATH"
256+
for wheel in ./wheelhouse/*.whl; do
257+
delvewheel repair "$wheel" --add-path "$EXTRA_DLL_PATH" -w wheelhouse/audited
258+
done
246259
- name: Archive wheel artifacts
247260
uses: actions/upload-artifact@v4
248261
with:
249262
name: audited-wheels-windows-${{ matrix.python-version }}
250263
path: bindings/python/wheelhouse/audited/*
251-
- name: Deploy wheels to artifactory (if not a release)
252-
if: startsWith(github.ref, 'refs/tags/v') != true
253-
run: cd bindings/python && bash ./ci/upload-artifactory.sh
254-
env:
255-
ARTIFACTORY_URL: ${{ secrets.ARTIFACTORY_URL }}
256-
ARTIFACTORY_USER: ${{ secrets.ARTIFACTORY_USER }}
257-
ARTIFACTORY_PASS: ${{ secrets.ARTIFACTORY_PASS }}
258264

259265
# This job builds wheels for macOS arch
260266
build-macos:
@@ -328,8 +334,28 @@ jobs:
328334
with:
329335
name: audited-wheels-macos-${{ matrix.os }}-${{ matrix.python-version }}
330336
path: bindings/python/wheelhouse/audited/*
331-
- name: Deploy wheels to artifactory (if not a release)
332-
if: startsWith(github.ref, 'refs/tags/v') != true
337+
338+
combine-macos-wheels:
339+
needs: build-macos
340+
strategy:
341+
matrix:
342+
os: [macos-13, macos-14]
343+
fail-fast: false
344+
runs-on: ${{ matrix.os }}
345+
steps:
346+
- uses: actions/checkout@v3
347+
- name: Download audited wheels
348+
uses: actions/download-artifact@v4
349+
with:
350+
pattern: audited-wheels-macos-${{ matrix.os }}-*
351+
path: bindings/python/wheelhouse/audited/
352+
merge-multiple: true
353+
- name: Combine wheels
354+
run: |
355+
cd bindings/python && mv wheelhouse/audited wheelhouse/audited_pre && mkdir -p wheelhouse/audited
356+
echo "Combining repaired wheels into one master wheel"
357+
python3 ci/combine_wheels.py --input_folder=wheelhouse/audited_pre --output_folder=wheelhouse/audited
358+
- name: Upload combined wheels to artifactory
333359
run: cd bindings/python && bash ./ci/upload-artifactory.sh
334360
env:
335361
ARTIFACTORY_URL: ${{ secrets.ARTIFACTORY_URL }}
@@ -403,8 +429,28 @@ jobs:
403429
with:
404430
name: audited-wheels-linux-x86_64-${{ matrix.python-set }}
405431
path: bindings/python/wheelhouse/audited/*
406-
- name: Deploy wheels to artifactory (if not a release)
407-
if: startsWith(github.ref, 'refs/tags/v') != true
432+
433+
combine-linux-x86_64-wheels:
434+
needs: build-linux-x86_64
435+
runs-on: ubuntu-latest
436+
container:
437+
image: quay.io/pypa/manylinux_2_28_x86_64:2024.12.05-1
438+
env:
439+
PLAT: manylinux_2_28_x86_64
440+
steps:
441+
- uses: actions/checkout@v3
442+
- name: Download audited wheels
443+
uses: actions/download-artifact@v4
444+
with:
445+
pattern: audited-wheels-linux-x86_64-*
446+
path: bindings/python/wheelhouse/audited/
447+
merge-multiple: true
448+
- name: Combine wheels
449+
run: |
450+
cd bindings/python && mv wheelhouse/audited wheelhouse/audited_pre && mkdir -p wheelhouse/audited
451+
echo "Combining repaired wheels into one master wheel"
452+
python3 ci/combine_wheels.py --input_folder=wheelhouse/audited_pre --output_folder=wheelhouse/audited
453+
- name: Upload combined wheels to artifactory
408454
run: cd bindings/python && bash ./ci/upload-artifactory.sh
409455
env:
410456
ARTIFACTORY_URL: ${{ secrets.ARTIFACTORY_URL }}
@@ -482,8 +528,59 @@ jobs:
482528
with:
483529
name: audited-wheels-linux-arm64-${{ matrix.python-set }}
484530
path: bindings/python/wheelhouse/audited/*
485-
- name: Deploy wheels to artifactory (if not a release)
486-
if: startsWith(github.ref, 'refs/tags/v') != true
531+
532+
combine-linux-arm64-wheels:
533+
needs: build-linux-arm64
534+
runs-on: ubuntu-24.04-arm
535+
timeout-minutes: 1440 # Set timeout to 24 hours
536+
container:
537+
image: quay.io/pypa/manylinux_2_28_aarch64:2024.12.05-1
538+
env:
539+
PLAT: manylinux_2_28_aarch64
540+
steps:
541+
- uses: actions/checkout@v3
542+
- name: Download audited wheels
543+
uses: actions/download-artifact@v4
544+
with:
545+
pattern: audited-wheels-linux-arm64-*
546+
path: bindings/python/wheelhouse/audited/
547+
merge-multiple: true
548+
- name: Combine wheels
549+
run: |
550+
cd bindings/python && mv wheelhouse/audited wheelhouse/audited_pre && mkdir -p wheelhouse/audited
551+
echo "Combining repaired wheels into one master wheel"
552+
python3 ci/combine_wheels.py --input_folder=wheelhouse/audited_pre --output_folder=wheelhouse/audited
553+
- name: Upload combined wheels to artifactory
554+
run: cd bindings/python && bash ./ci/upload-artifactory.sh
555+
env:
556+
ARTIFACTORY_URL: ${{ secrets.ARTIFACTORY_URL }}
557+
ARTIFACTORY_USER: ${{ secrets.ARTIFACTORY_USER }}
558+
ARTIFACTORY_PASS: ${{ secrets.ARTIFACTORY_PASS }}
559+
560+
combine-windows-x86_64-wheels:
561+
needs: build-windows-x86_64
562+
runs-on: windows-latest
563+
steps:
564+
- uses: actions/checkout@v3
565+
- name: Download audited wheels
566+
uses: actions/download-artifact@v4
567+
with:
568+
pattern: audited-wheels-windows-*
569+
path: bindings/python/wheelhouse/audited/
570+
merge-multiple: true
571+
- name: Set up Python
572+
uses: actions/setup-python@v4
573+
with:
574+
python-version: "3.12"
575+
- name: Combine wheels
576+
run: |
577+
python -m pip install delvewheel # Install delvewheel for patching wheels
578+
cd bindings/python
579+
mv wheelhouse/audited wheelhouse/audited_pre
580+
mkdir -p wheelhouse/audited
581+
echo "Combining repaired wheels into one master wheel"
582+
python ci/combine_wheels.py --input_folder=wheelhouse/audited_pre --output_folder=wheelhouse/audited --log_level=debug
583+
- name: Upload combined wheels to artifactory
487584
run: cd bindings/python && bash ./ci/upload-artifactory.sh
488585
env:
489586
ARTIFACTORY_URL: ${{ secrets.ARTIFACTORY_URL }}
@@ -494,7 +591,7 @@ jobs:
494591
if: startsWith(github.ref, 'refs/tags/v')
495592
# needs: [pytest, build-linux-armhf, build-windows-x86_64, build-macos-x86_64, build-macos-arm64, build-linux-x86_64, build-linux-arm64]
496593
# needs: [pytest, build-windows-x86_64, build-macos, build-linux-x86_64, build-linux-arm64]
497-
needs: [build-windows-x86_64, build-macos, build-linux-x86_64, build-linux-arm64]
594+
needs: [combine-macos-wheels, combine-linux-x86_64-wheels, combine-linux-arm64-wheels, combine-windows-x86_64-wheels]
498595
runs-on: ubuntu-latest
499596

500597
steps:

0 commit comments

Comments
 (0)