Skip to content

Commit f023db8

Browse files
NicolasHugahmadsharif1
NicolasHug
and
ahmadsharif1
authored
Build wheels on test-infra machines (#160)
Co-authored-by: Ahmad Sharif <[email protected]>
1 parent 18eb3fd commit f023db8

File tree

4 files changed

+93
-74
lines changed

4 files changed

+93
-74
lines changed

‎.github/workflows/wheel.yaml‎

Lines changed: 45 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -1,102 +1,73 @@
1-
name: Build wheel
1+
name: Build and test Linux wheels
22

33
on:
4-
push:
5-
branches: [ main ]
64
pull_request:
5+
push:
6+
branches:
7+
- nightly
8+
- main
9+
- release/*
10+
tags:
11+
- v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+
12+
workflow_dispatch:
713

814
concurrency:
9-
group: unit-test${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_number || github.ref }}
15+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ github.ref_type == 'branch' && github.sha }}-${{ github.event_name == 'workflow_dispatch' }}
1016
cancel-in-progress: true
1117

18+
permissions:
19+
id-token: write
20+
contents: write
21+
1222
defaults:
1323
run:
1424
shell: bash -l -eo pipefail {0}
1525

1626
jobs:
27+
generate-matrix:
28+
uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main
29+
with:
30+
package-type: wheel
31+
os: linux
32+
test-infra-repository: pytorch/test-infra
33+
test-infra-ref: main
34+
with-xpu: disable
35+
with-rocm: disable
36+
with-cuda: disable
1737
build:
18-
runs-on: ubuntu-20.04
38+
needs: generate-matrix
1939
strategy:
2040
fail-fast: false
21-
matrix:
22-
python-version: ['3.8', '3.12']
23-
steps:
24-
- name: Check out repo
25-
uses: actions/checkout@v3
26-
- name: Setup conda env
27-
uses: conda-incubator/setup-miniconda@v2
28-
with:
29-
auto-update-conda: true
30-
miniconda-version: "latest"
31-
activate-environment: test
32-
python-version: ${{ matrix.python-version }}
33-
- name: Update pip
34-
run: python -m pip install --upgrade pip
35-
- name: Install Pytorch
36-
run: |
37-
python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu
38-
- name: Check GCC version
39-
run: |
40-
# We want to build wheels on gcc 9 to be consistent with the torch
41-
# binaries, which is why we're using ubuntu-20.04 and not
42-
# ubuntu-latest. More details below about the GLIBCXX checks.
43-
if ! gcc --version | grep -q -E "9\.[0-9]+\.[0-9]+"
44-
then
45-
echo "Wrong GCC version, exiting!"
46-
gcc --version
47-
exit 1
48-
fi
49-
- name: Build the torchcodec wheel
50-
run: |
51-
# Just for sanity, make sure FFmpeg isn't installed or needed for buidling.
52-
.github/scripts/assert_ffmpeg_not_installed.sh
53-
54-
python -m pip install build
55-
BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 python -m build . -vvv --no-isolation
56-
- name: Validate wheel content
57-
run: |
58-
source .github/scripts/helpers.sh
59-
wheel_path=$(pwd)/$(find dist -type f -name "*.whl")
60-
echo "Wheel content:"
61-
unzip -l $wheel_path
62-
63-
for ffmpeg_major_version in 4 5 6 7; do
64-
assert_in_wheel $wheel_path torchcodec/libtorchcodec${ffmpeg_major_version}.so
65-
done
66-
assert_not_in_wheel $wheel_path libtorchcodec.so
67-
68-
for ffmpeg_so in libavcodec.so libavfilter.so libavformat.so libavutil.so libavdevice.so ; do
69-
assert_not_in_wheel $wheel_path $ffmpeg_so
70-
done
71-
72-
assert_not_in_wheel $wheel_path "^test"
73-
assert_not_in_wheel $wheel_path "^doc"
74-
assert_not_in_wheel $wheel_path "^benchmarks"
75-
76-
# See invoked python script below for details about this check.
77-
extracted_wheel_dir=$(mktemp -d)
78-
unzip -q $wheel_path -d $extracted_wheel_dir
79-
symbols_matches=$(find $extracted_wheel_dir | grep ".so$" | xargs objdump --syms | grep GLIBCXX_3.4.)
80-
python .github/scripts/check_glibcxx.py "$symbols_matches"
81-
- uses: actions/upload-artifact@v4
82-
with:
83-
name: sdist-and-wheel-linux_x86_${{ matrix.python-version }}
84-
path: dist/*
41+
name: Build and Upload wheel
42+
uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main
43+
with:
44+
repository: pytorch/torchcodec
45+
ref: ""
46+
test-infra-repository: pytorch/test-infra
47+
test-infra-ref: main
48+
build-matrix: ${{ needs.generate-matrix.outputs.matrix }}
49+
post-script: packaging/post_build_script.sh
50+
env-var-script: packaging/env_var_script.sh
51+
smoke-test-script: packaging/fake_smoke_test.py
52+
package-name: torchcodec
53+
trigger-event: ${{ github.event_name }}
54+
build-platform: "python-build-package"
55+
wheel-build-extra-args: "-vvv --no-isolation"
8556

8657
install-and-test:
8758
runs-on: ubuntu-latest
8859
strategy:
8960
fail-fast: false
9061
matrix:
91-
python-version: ['3.8', '3.12']
62+
python-version: ['3.9']
9263
ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1']
9364
if: ${{ always() }}
9465
needs: build
9566
steps:
96-
- uses: actions/download-artifact@v4
67+
- uses: actions/download-artifact@v3
9768
with:
98-
name: sdist-and-wheel-linux_x86_${{ matrix.python-version }}
99-
path: dist/
69+
name: pytorch_torchcodec__${{ matrix.python-version }}_cpu_x86_64
70+
path: pytorch/torchcodec/dist/
10071
- name: Setup conda env
10172
uses: conda-incubator/setup-miniconda@v2
10273
with:
@@ -111,7 +82,7 @@ jobs:
11182
python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu
11283
- name: Install torchcodec from the wheel
11384
run: |
114-
wheel_path=`find dist -type f -name "*.whl"`
85+
wheel_path=`find pytorch/torchcodec/dist -type f -name "*.whl"`
11586
echo Installing $wheel_path
11687
python -m pip install $wheel_path -vvv
11788

‎packaging/env_var_script.sh‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
#!/bin/bash
2+
3+
export BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1

‎packaging/fake_smoke_test.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
# This is a fake smoke test that runs on the test-infra instances after we build
2+
# a wheel. We cannot run a real smoke test over there, because the machines are
3+
# too old to even install a proper ffmpeg version - and without ffmpeg,
4+
# importing torchcodec just fails. It's OK, we run our *entire* test suite on
5+
# those wheels anyway (on other machines).
6+
7+
print("Success")

‎packaging/post_build_script.sh‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
#!/bin/bash
2+
3+
source .github/scripts/helpers.sh
4+
5+
wheel_path=$(pwd)/$(find dist -type f -name "*.whl")
6+
echo "Wheel content:"
7+
unzip -l $wheel_path
8+
9+
for ffmpeg_major_version in 4 5 6 7; do
10+
assert_in_wheel $wheel_path torchcodec/libtorchcodec${ffmpeg_major_version}.so
11+
done
12+
assert_not_in_wheel $wheel_path libtorchcodec.so
13+
14+
for ffmpeg_so in libavcodec.so libavfilter.so libavformat.so libavutil.so libavdevice.so ; do
15+
assert_not_in_wheel $wheel_path $ffmpeg_so
16+
done
17+
18+
assert_not_in_wheel $wheel_path "^test"
19+
assert_not_in_wheel $wheel_path "^doc"
20+
assert_not_in_wheel $wheel_path "^benchmarks"
21+
assert_not_in_wheel $wheel_path "^packaging"
22+
23+
# See invoked python script below for details about this check.
24+
extracted_wheel_dir=$(mktemp -d)
25+
unzip -q $wheel_path -d $extracted_wheel_dir
26+
symbols_matches=$(find $extracted_wheel_dir | grep ".so$" | xargs objdump --syms | grep GLIBCXX_3.4.)
27+
python .github/scripts/check_glibcxx.py "$symbols_matches"
28+
29+
echo "ls dist"
30+
ls dist
31+
32+
old="linux_x86_64"
33+
new="manylinux_2_17_x86_64.manylinux2014_x86_64"
34+
echo "Replacing ${old} with ${new} in wheel name"
35+
mv dist/*${old}*.whl $(echo dist/*${old}*.whl | sed "s/${old}/${new}/")
36+
37+
echo "ls dist"
38+
ls dist

0 commit comments

Comments
 (0)