Skip to content

Commit 5d0fd55

Browse files
scottsscotts
committed
Merge branch 'mac_wheels_ci' of github.com:scotts/torchcodec into mac_wheels_ci
2 parents 2733862 + 4a8eb56 commit 5d0fd55

File tree

2 files changed

+113
-49
lines changed

2 files changed

+113
-49
lines changed

‎.github/workflows/macos_wheel.yaml‎

Lines changed: 105 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -54,16 +54,6 @@ jobs:
5454
build-platform: "python-build-package"
5555
build-command: "BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 ${CONDA_RUN} python3 -m build --wheel -vvv --no-isolation"
5656

57-
validate-binaries:
58-
uses: pytorch/test-infra/.github/workflows/validate-domain-library.yml@main
59-
with:
60-
package_type: "wheel"
61-
os: "macos-arm64"
62-
channel: "nightly"
63-
repository: "pytorch/torchcodec"
64-
smoke_test: "source ./packaging/validate_wheel.sh"
65-
install_torch: true
66-
6757
install-and-test:
6858
runs-on: macos-m1-stable
6959
strategy:
@@ -74,65 +64,131 @@ jobs:
7464
if: ${{ always() }}
7565
needs: build
7666
steps:
77-
- uses: actions/download-artifact@v3
78-
with:
79-
name: pytorch_torchcodec__${{ matrix.python-version }}_cpu_
80-
path: pytorch/torchcodec/dist/
8167
- name: Setup miniconda
8268
uses: pytorch/test-infra/.github/actions/setup-miniconda@macbuildwheel
8369
with:
8470
python-version: ${{ matrix.python-version }}
85-
- name: Update pip
71+
- name: Check out torchcodec repo
72+
uses: actions/checkout@v3
73+
- name: Check out tet-infra repo
74+
uses: actions/checkout@v3
75+
with:
76+
repository: pytorch/test-infra
77+
ref: macbuildwheel
78+
path: test-infra
79+
- name: Download wheel
80+
uses: actions/download-artifact@v3
81+
with:
82+
name: pytorch_torchcodec__${{ matrix.python-version }}_cpu_
83+
path: pytorch/torchcodec/dist/
84+
85+
## start copying from test-infra/.github/actions/setup-binary-builds/action.yaml
86+
- name: Clean conda environment
87+
shell: bash -l {0}
88+
run: |
89+
set -euxo pipefail
90+
conda info | grep -i 'base environment'
91+
conda clean --all --quiet --yes
92+
- name: Reset channel priority
93+
shell: bash -l {0}
94+
run: |
95+
set -euxo pipefail
96+
conda config --set channel_priority false
97+
- name: Discover dir structure
98+
shell: bash -l {0}
8699
run: |
87-
python3 -m pip install --upgrade pip
100+
echo "pwd"
101+
pwd
102+
echo "ls -lh"
103+
ls -lh
104+
echo "ls -lh pytorch/torchcodec"
105+
ls -lh pytorch/torchcodec
106+
echo "ls -lh pytorch/torchcodec/dist"
107+
ls -lh pytorch/torchcodec/dist
108+
wheel_path=`find pytorch/torchcodec/dist -type f -name "*.whl"`
109+
echo "unzip $wheel_path -d unzipped_wheel"
110+
unzip $wheel_path -d unzipped_wheel
111+
echo "ls -lh unzipped_wheel"
112+
ls -lh unzipped_wheel
113+
- name: Generate file from pytorch_pkg_helpers
114+
# below does not seem necessary in our context
115+
#working-directory: "pytorch/torchcodec"
116+
shell: bash -l {0}
117+
run: |
118+
set -euxo pipefail
119+
CONDA_ENV="${RUNNER_TEMP}/pytorch_pkg_helpers_${GITHUB_RUN_ID}"
120+
conda create \
121+
--yes --quiet \
122+
--prefix "${CONDA_ENV}" \
123+
"python=3.9"
124+
CONDA_ENV="${CONDA_ENV}"
125+
CONDA_RUN="conda run -p ${CONDA_ENV}"
126+
${CONDA_RUN} python -m pip install ${GITHUB_WORKSPACE}/test-infra/tools/pkg-helpers
127+
BUILD_ENV_FILE="${RUNNER_TEMP}/build_env_${GITHUB_RUN_ID}"
128+
${CONDA_RUN} python -m pytorch_pkg_helpers > "${BUILD_ENV_FILE}"
129+
cat "${BUILD_ENV_FILE}"
130+
echo "BUILD_ENV_FILE=${BUILD_ENV_FILE}" >> "${GITHUB_ENV}"
131+
- name: Setup conda environment for build
132+
shell: bash -l {0}
133+
env:
134+
PYTHON_VERSION: ${{ matrix.python-version }}
135+
run: |
136+
set -euxo pipefail
137+
CONDA_ENV="${RUNNER_TEMP}/conda_environment_${GITHUB_RUN_ID}"
138+
if [[ "${PACKAGE_TYPE:-}" == "conda" ]]; then
139+
# For conda package host python version is irrelevant
140+
export PYTHON_VERSION=3.9
141+
export CONDA_BUILD_EXTRA="conda=24.4.0 conda-build=24.3.0 python-libarchive-c=2.9"
142+
else
143+
# For wheel builds we don't need neither conda nor conda-build
144+
export CONDA_BUILD_EXTRA=""
145+
fi
146+
147+
conda create \
148+
--yes --quiet \
149+
--prefix "${CONDA_ENV}" \
150+
"python=${PYTHON_VERSION}" \
151+
cmake=3.26 \
152+
ninja=1.10 \
153+
pkg-config=0.29 \
154+
${CONDA_BUILD_EXTRA} \
155+
wheel=0.37
156+
157+
echo "CONDA_ENV=${CONDA_ENV}" >> "${GITHUB_ENV}"
158+
echo "CONDA_RUN=conda run -p ${CONDA_ENV}" >> "${GITHUB_ENV}"
159+
## end copying
160+
- name: Run otool
161+
shell: bash -l {0}
162+
run: |
163+
echo "otool -L unzipped_wheel/torchcodec/libtorchcodec*"
164+
otool -L unzipped_wheel/torchcodec/libtorchcodec*
165+
166+
- name: Update pip
167+
run: python -m pip install --upgrade pip
88168
- name: Install PyTorch
89169
run: |
90-
python3 -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu
170+
${CONDA_RUN} python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu
91171
- name: Install torchcodec from the wheel
92172
run: |
93173
wheel_path=`find pytorch/torchcodec/dist -type f -name "*.whl"`
94174
echo Installing $wheel_path
95-
python3 -m pip install $wheel_path -vvv
96-
97-
- name: Check out repo
98-
uses: actions/checkout@v3
175+
${CONDA_RUN} python -m pip install $wheel_path -vvv
99176
- name: Install ffmpeg, post build
100177
run: |
101-
# Ideally we would have checked for that before installing the wheel,
102-
# but we need to checkout the repo to access this file, and we don't
103-
# want to checkout the repo before installing the wheel to avoid any
104-
# side-effect. It's OK.
105-
source packaging/helpers.sh
106-
107-
# below was failing, but when I just try to call "ffmpeg -version" that also fails?
108-
#assert_ffmpeg_not_installed
109-
110-
conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge
111-
ffmpeg -version
112-
178+
${CONDA_RUN} conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge
179+
${CONDA_RUN} ffmpeg -version
113180
- name: Install test dependencies
114181
run: |
115-
python3 -m pip install --pre torchvision --index-url https://download.pytorch.org/whl/nightly/cpu
116-
# Ideally we would find a way to get those dependencies from pyproject.toml
117-
python3 -m pip install numpy pytest pillow
118-
182+
${CONDA_RUN} python -m pip install --pre torchvision --index-url https://download.pytorch.org/whl/nightly/cpu
183+
${CONDA_RUN} python -m pip install numpy pytest pillow
119184
- name: Delete the src/ folder just for fun
120185
run: |
121-
# The only reason we checked-out the repo is to get access to the
122-
# tests. We don't care about the rest. Out of precaution, we delete
123-
# the src/ folder to be extra sure that we're running the code from
124-
# the installed wheel rather than from the source.
125-
# This is just to be extra cautious and very overkill because a)
126-
# there's no way the `torchcodec` package from src/ can be found from
127-
# the PythonPath: the main point of `src/` is precisely to protect
128-
# against that and b) if we ever were to execute code from
129-
# `src/torchcodec`, it would fail loudly because the built .so files
130-
# aren't present there.
131186
rm -r src/
132187
ls
133188
- name: Smoke test
134189
run: |
135-
python3 test/decoders/manual_smoke_test.py
190+
${CONDA_RUN} lldb -f python -- test/decoders/manual_smoke_test.py
191+
ls -lh
136192
- name: Run Python tests
137193
run: |
138-
pytest test
194+
${CONDA_RUN} pytest test

‎packaging/validate_wheel.sh‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,16 @@
33
set -eux
44

55
conda install ffmpeg -c conda-forge
6+
7+
# line below is inspired by:
8+
# https://github.com/pytorch/text/blob/1d4ce73c57417f1af8278af56631de3c25e3bbaf/.github/scripts/validate_binaries.sh#L5
9+
# but when looking at the actual job that runs, we seem to install 0.0.0dev, a version
10+
# we probably uploaded during release testing. How do we get the wheel generated during
11+
# the build job?
612
pip install ${PYTORCH_PIP_PREFIX} torchcodec --extra-index-url ${PYTORCH_PIP_DOWNLOAD_URL}
713
pip install --pre torchvision --index-url https://download.pytorch.org/whl/nightly/cpu
14+
pip install numpy pytest pillow
815

916
python3 test/decoders/manual_smoke_test.py
17+
1018
pytest test -vvv

0 commit comments

Comments
 (0)