Skip to content

Commit 952ca7d

Browse files
NicolasHugNicolasHug
committed
Merge branch 'main' of github.com:pytorch/torchcodec into add-windows-cmake
2 parents edca0c3 + 3c865b2 commit 952ca7d

39 files changed

+1598
-374
lines changed

‎.github/workflows/build_ffmpeg.yaml‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,3 +70,25 @@ jobs:
7070
artifact_dir="${RUNNER_ARTIFACT_DIR}/$(date +%Y-%m-%d)/macos_$(uname -m)"
7171
mkdir -p "${artifact_dir}"
7272
mv ffmpeg.tar.gz "${artifact_dir}/${FFMPEG_VERSION}.tar.gz"
73+
74+
LGPL-Windows:
75+
strategy:
76+
fail-fast: false
77+
matrix:
78+
ffmpeg-version: ["4.4.4", "5.1.4", "6.1.1", "7.0.1"]
79+
uses: pytorch/test-infra/.github/workflows/windows_job.yml@main
80+
with:
81+
job-name: Build
82+
upload-artifact: ffmpeg-lgpl-windows_x86_64-${{ matrix.ffmpeg-version }}
83+
repository: pytorch/torchcodec
84+
script: |
85+
export FFMPEG_VERSION="${{ matrix.ffmpeg-version }}"
86+
export FFMPEG_ROOT="${PWD}/ffmpeg"
87+
88+
packaging/build_ffmpeg.bat
89+
90+
tar -cf ffmpeg.tar.gz ffmpeg/include ffmpeg/bin
91+
92+
artifact_dir="${RUNNER_ARTIFACT_DIR}/$(date +%Y-%m-%d)/windows_$(uname -m)"
93+
mkdir -p "${artifact_dir}"
94+
mv ffmpeg.tar.gz "${artifact_dir}/${FFMPEG_VERSION}.tar.gz"

‎.github/workflows/linux_cuda_wheel.yaml‎

Lines changed: 166 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
name: Build and test Linux CUDA wheels
22

33
on:
4-
pull_request:
4+
# pull_request:
55
push:
66
branches:
77
- nightly
@@ -56,6 +56,7 @@ jobs:
5656
build-platform: "python-build-package"
5757
build-command: "BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 ENABLE_CUDA=1 python -m build --wheel -vvv --no-isolation"
5858

59+
<<<<<<< HEAD
5960
# install-and-test:
6061
# runs-on: linux.g5.4xlarge.nvidia.gpu
6162
# strategy:
@@ -70,7 +71,41 @@ jobs:
7071
# cuda-version: ['12.6', '12.8']
7172
# # TODO: put back ffmpeg 5 https://github.com/pytorch/torchcodec/issues/325
7273
# ffmpeg-version-for-tests: ['4.4.2', '6', '7']
74+
||||||| 7945e6a
75+
install-and-test:
76+
runs-on: linux.g5.4xlarge.nvidia.gpu
77+
strategy:
78+
fail-fast: false
79+
matrix:
80+
# 3.9 corresponds to the minimum python version for which we build
81+
# the wheel unless the label cliflow/binaries/all is present in the
82+
# PR.
83+
# For the actual release we should add that label and change this to
84+
# include more python versions.
85+
python-version: ['3.9']
86+
cuda-version: ['12.6', '12.8']
87+
# TODO: put back ffmpeg 5 https://github.com/pytorch/torchcodec/issues/325
88+
ffmpeg-version-for-tests: ['4.4.2', '6', '7']
89+
=======
90+
install-and-test:
91+
runs-on: linux.g5.4xlarge.nvidia.gpu
92+
strategy:
93+
fail-fast: false
94+
matrix:
95+
# 3.9 corresponds to the minimum python version for which we build
96+
# the wheel unless the label cliflow/binaries/all is present in the
97+
# PR.
98+
# For the actual release we should add that label and change this to
99+
# include more python versions.
100+
python-version: ['3.9']
101+
# We test against 12.6 and 12.9 to avoid having too big of a CI matrix,
102+
# but for releases we should add 12.8.
103+
cuda-version: ['12.6', '12.9']
104+
# TODO: put back ffmpeg 5 https://github.com/pytorch/torchcodec/issues/325
105+
ffmpeg-version-for-tests: ['4.4.2', '6', '7']
106+
>>>>>>> 3c865b251bc90678621d5249083ea651c477ee8d
73107

108+
<<<<<<< HEAD
74109
# container:
75110
# image: "pytorch/manylinux2_28-builder:cuda${{ matrix.cuda-version }}"
76111
# options: "--gpus all -e NVIDIA_DRIVER_CAPABILITIES=video,compute,utility"
@@ -112,6 +147,92 @@ jobs:
112147
# wheel_path=`find pytorch/torchcodec/dist -type f -name "*cu${{ env.cuda_version_without_periods }}-cp${{ env.python_version_without_periods }}*.whl"`
113148
# echo Installing $wheel_path
114149
# ${CONDA_RUN} python -m pip install $wheel_path -vvv
150+
||||||| 7945e6a
151+
container:
152+
image: "pytorch/manylinux2_28-builder:cuda${{ matrix.cuda-version }}"
153+
options: "--gpus all -e NVIDIA_DRIVER_CAPABILITIES=video,compute,utility"
154+
needs: build
155+
steps:
156+
- name: Setup env vars
157+
run: |
158+
cuda_version_without_periods=$(echo "${{ matrix.cuda-version }}" | sed 's/\.//g')
159+
echo cuda_version_without_periods=${cuda_version_without_periods} >> $GITHUB_ENV
160+
python_version_without_periods=$(echo "${{ matrix.python-version }}" | sed 's/\.//g')
161+
echo python_version_without_periods=${python_version_without_periods} >> $GITHUB_ENV
162+
- uses: actions/download-artifact@v4
163+
with:
164+
name: pytorch_torchcodec__${{ matrix.python-version }}_cu${{ env.cuda_version_without_periods }}_x86_64
165+
path: pytorch/torchcodec/dist/
166+
- name: Setup miniconda using test-infra
167+
uses: pytorch/test-infra/.github/actions/setup-miniconda@main
168+
with:
169+
python-version: ${{ matrix.python-version }}
170+
# We install conda packages at the start because otherwise conda may have conflicts with dependencies.
171+
default-packages: "nvidia/label/cuda-${{ matrix.cuda-version }}.0::libnpp nvidia::cuda-nvrtc=${{ matrix.cuda-version }} nvidia::cuda-toolkit=${{ matrix.cuda-version }} nvidia::cuda-cudart=${{ matrix.cuda-version }} nvidia::cuda-driver-dev=${{ matrix.cuda-version }} conda-forge::ffmpeg=${{ matrix.ffmpeg-version-for-tests }}"
172+
- name: Check env
173+
run: |
174+
${CONDA_RUN} env
175+
${CONDA_RUN} conda info
176+
${CONDA_RUN} nvidia-smi
177+
${CONDA_RUN} conda list
178+
- name: Assert ffmpeg exists
179+
run: |
180+
${CONDA_RUN} ffmpeg -buildconf
181+
- name: Update pip
182+
run: ${CONDA_RUN} python -m pip install --upgrade pip
183+
- name: Install PyTorch
184+
run: |
185+
${CONDA_RUN} python -m pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cu${{ env.cuda_version_without_periods }}
186+
${CONDA_RUN} python -c 'import torch; print(f"{torch.__version__}"); print(f"{torch.__file__}"); print(f"{torch.cuda.is_available()=}")'
187+
- name: Install torchcodec from the wheel
188+
run: |
189+
wheel_path=`find pytorch/torchcodec/dist -type f -name "*cu${{ env.cuda_version_without_periods }}-cp${{ env.python_version_without_periods }}*.whl"`
190+
echo Installing $wheel_path
191+
${CONDA_RUN} python -m pip install $wheel_path -vvv
192+
=======
193+
container:
194+
image: "pytorch/manylinux2_28-builder:cuda${{ matrix.cuda-version }}"
195+
options: "--gpus all -e NVIDIA_DRIVER_CAPABILITIES=video,compute,utility"
196+
needs: build
197+
steps:
198+
- name: Setup env vars
199+
run: |
200+
cuda_version_without_periods=$(echo "${{ matrix.cuda-version }}" | sed 's/\.//g')
201+
echo cuda_version_without_periods=${cuda_version_without_periods} >> $GITHUB_ENV
202+
python_version_without_periods=$(echo "${{ matrix.python-version }}" | sed 's/\.//g')
203+
echo python_version_without_periods=${python_version_without_periods} >> $GITHUB_ENV
204+
- uses: actions/download-artifact@v4
205+
with:
206+
name: pytorch_torchcodec__${{ matrix.python-version }}_cu${{ env.cuda_version_without_periods }}_x86_64
207+
path: pytorch/torchcodec/dist/
208+
- name: Setup miniconda using test-infra
209+
uses: pytorch/test-infra/.github/actions/setup-miniconda@main
210+
with:
211+
python-version: ${{ matrix.python-version }}
212+
# We install conda packages at the start because otherwise conda may have conflicts with dependencies.
213+
# Note: xorg-libxau was addded to fix a problem with ffmpeg 4. We should consider removing it.
214+
default-packages: "nvidia/label/cuda-${{ matrix.cuda-version }}.0::libnpp nvidia::cuda-nvrtc=${{ matrix.cuda-version }} nvidia::cuda-toolkit=${{ matrix.cuda-version }} nvidia::cuda-cudart=${{ matrix.cuda-version }} nvidia::cuda-driver-dev=${{ matrix.cuda-version }} conda-forge::ffmpeg=${{ matrix.ffmpeg-version-for-tests }} conda-forge::xorg-libxau"
215+
- name: Check env
216+
run: |
217+
${CONDA_RUN} env
218+
${CONDA_RUN} conda info
219+
${CONDA_RUN} nvidia-smi
220+
${CONDA_RUN} conda list
221+
- name: Assert ffmpeg exists
222+
run: |
223+
${CONDA_RUN} ffmpeg -buildconf
224+
- name: Update pip
225+
run: ${CONDA_RUN} python -m pip install --upgrade pip
226+
- name: Install PyTorch
227+
run: |
228+
${CONDA_RUN} python -m pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cu${{ env.cuda_version_without_periods }}
229+
${CONDA_RUN} python -c 'import torch; print(f"{torch.__version__}"); print(f"{torch.__file__}"); print(f"{torch.cuda.is_available()=}")'
230+
- name: Install torchcodec from the wheel
231+
run: |
232+
wheel_path=`find pytorch/torchcodec/dist -type f -name "*cu${{ env.cuda_version_without_periods }}-cp${{ env.python_version_without_periods }}*.whl"`
233+
echo Installing $wheel_path
234+
${CONDA_RUN} python -m pip install $wheel_path -vvv
235+
>>>>>>> 3c865b251bc90678621d5249083ea651c477ee8d
115236

116237
# - name: Check out repo
117238
# uses: actions/checkout@v3
@@ -121,6 +242,7 @@ jobs:
121242
# # Ideally we would find a way to get those dependencies from pyproject.toml
122243
# ${CONDA_RUN} python -m pip install numpy pytest pillow
123244

245+
<<<<<<< HEAD
124246
# - name: Delete the src/ folder just for fun
125247
# run: |
126248
# # The only reason we checked-out the repo is to get access to the
@@ -141,3 +263,46 @@ jobs:
141263
# - name: Run Python benchmark
142264
# run: |
143265
# ${CONDA_RUN} time python benchmarks/decoders/gpu_benchmark.py --devices=cuda:0,cpu --resize_devices=none
266+
||||||| 7945e6a
267+
- name: Delete the src/ folder just for fun
268+
run: |
269+
# The only reason we checked-out the repo is to get access to the
270+
# tests. We don't care about the rest. Out of precaution, we delete
271+
# the src/ folder to be extra sure that we're running the code from
272+
# the installed wheel rather than from the source.
273+
# This is just to be extra cautious and very overkill because a)
274+
# there's no way the `torchcodec` package from src/ can be found from
275+
# the PythonPath: the main point of `src/` is precisely to protect
276+
# against that and b) if we ever were to execute code from
277+
# `src/torchcodec`, it would fail loudly because the built .so files
278+
# aren't present there.
279+
rm -r src/
280+
ls
281+
- name: Run Python tests
282+
run: |
283+
${CONDA_RUN} FAIL_WITHOUT_CUDA=1 pytest test -v --tb=short
284+
- name: Run Python benchmark
285+
run: |
286+
${CONDA_RUN} time python benchmarks/decoders/gpu_benchmark.py --devices=cuda:0,cpu --resize_devices=none
287+
=======
288+
- name: Delete the src/ folder just for fun
289+
run: |
290+
# The only reason we checked-out the repo is to get access to the
291+
# tests. We don't care about the rest. Out of precaution, we delete
292+
# the src/ folder to be extra sure that we're running the code from
293+
# the installed wheel rather than from the source.
294+
# This is just to be extra cautious and very overkill because a)
295+
# there's no way the `torchcodec` package from src/ can be found from
296+
# the PythonPath: the main point of `src/` is precisely to protect
297+
# against that and b) if we ever were to execute code from
298+
# `src/torchcodec`, it would fail loudly because the built .so files
299+
# aren't present there.
300+
rm -r src/
301+
ls
302+
- name: Run Python tests
303+
run: |
304+
${CONDA_RUN} FAIL_WITHOUT_CUDA=1 pytest --override-ini="addopts=-v" test --tb=short
305+
- name: Run Python benchmark
306+
run: |
307+
${CONDA_RUN} time python benchmarks/decoders/gpu_benchmark.py --devices=cuda:0,cpu --resize_devices=none
308+
>>>>>>> 3c865b251bc90678621d5249083ea651c477ee8d

‎.github/workflows/linux_wheel.yaml‎

Lines changed: 64 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
name: Build and test Linux wheel
22

33
on:
4-
pull_request:
4+
# pull_request:
55
push:
66
branches:
77
- nightly
@@ -57,70 +57,70 @@ jobs:
5757
build-platform: "python-build-package"
5858
build-command: "BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 python -m build --wheel -vvv --no-isolation"
5959

60-
# install-and-test:
61-
# runs-on: ubuntu-latest
62-
# strategy:
63-
# fail-fast: false
64-
# matrix:
65-
# python-version: ['3.9']
66-
# ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1']
67-
# needs: build
68-
# steps:
69-
# - uses: actions/download-artifact@v4
70-
# with:
71-
# name: pytorch_torchcodec__${{ matrix.python-version }}_cpu_x86_64
72-
# path: pytorch/torchcodec/dist/
73-
# - name: Setup conda env
74-
# uses: conda-incubator/setup-miniconda@v2
75-
# with:
76-
# auto-update-conda: true
77-
# miniconda-version: "latest"
78-
# activate-environment: test
79-
# python-version: ${{ matrix.python-version }}
80-
# - name: Update pip
81-
# run: python -m pip install --upgrade pip
82-
# - name: Install PyTorch
83-
# run: |
84-
# python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu
85-
# - name: Install torchcodec from the wheel
86-
# run: |
87-
# wheel_path=`find pytorch/torchcodec/dist -type f -name "*.whl"`
88-
# echo Installing $wheel_path
89-
# python -m pip install $wheel_path -vvv
60+
install-and-test:
61+
runs-on: ubuntu-latest
62+
strategy:
63+
fail-fast: false
64+
matrix:
65+
python-version: ['3.9']
66+
ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1']
67+
needs: build
68+
steps:
69+
- uses: actions/download-artifact@v4
70+
with:
71+
name: pytorch_torchcodec__${{ matrix.python-version }}_cpu_x86_64
72+
path: pytorch/torchcodec/dist/
73+
- name: Setup conda env
74+
uses: conda-incubator/setup-miniconda@v2
75+
with:
76+
auto-update-conda: true
77+
miniconda-version: "latest"
78+
activate-environment: test
79+
python-version: ${{ matrix.python-version }}
80+
- name: Update pip
81+
run: python -m pip install --upgrade pip
82+
- name: Install PyTorch
83+
run: |
84+
python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu
85+
- name: Install torchcodec from the wheel
86+
run: |
87+
wheel_path=`find pytorch/torchcodec/dist -type f -name "*.whl"`
88+
echo Installing $wheel_path
89+
python -m pip install $wheel_path -vvv
9090
91-
# - name: Check out repo
92-
# uses: actions/checkout@v3
93-
# - name: Install ffmpeg, post build
94-
# run: |
95-
# # Ideally we would have checked for that before installing the wheel,
96-
# # but we need to checkout the repo to access this file, and we don't
97-
# # want to checkout the repo before installing the wheel to avoid any
98-
# # side-effect. It's OK.
99-
# source packaging/helpers.sh
100-
# assert_ffmpeg_not_installed
91+
- name: Check out repo
92+
uses: actions/checkout@v3
93+
- name: Install ffmpeg, post build
94+
run: |
95+
# Ideally we would have checked for that before installing the wheel,
96+
# but we need to checkout the repo to access this file, and we don't
97+
# want to checkout the repo before installing the wheel to avoid any
98+
# side-effect. It's OK.
99+
source packaging/helpers.sh
100+
assert_ffmpeg_not_installed
101101
102-
# conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge
103-
# ffmpeg -version
102+
conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge
103+
ffmpeg -version
104104
105-
# - name: Install test dependencies
106-
# run: |
107-
# # Ideally we would find a way to get those dependencies from pyproject.toml
108-
# python -m pip install numpy pytest pillow
105+
- name: Install test dependencies
106+
run: |
107+
# Ideally we would find a way to get those dependencies from pyproject.toml
108+
python -m pip install numpy pytest pillow
109109
110-
# - name: Delete the src/ folder just for fun
111-
# run: |
112-
# # The only reason we checked-out the repo is to get access to the
113-
# # tests. We don't care about the rest. Out of precaution, we delete
114-
# # the src/ folder to be extra sure that we're running the code from
115-
# # the installed wheel rather than from the source.
116-
# # This is just to be extra cautious and very overkill because a)
117-
# # there's no way the `torchcodec` package from src/ can be found from
118-
# # the PythonPath: the main point of `src/` is precisely to protect
119-
# # against that and b) if we ever were to execute code from
120-
# # `src/torchcodec`, it would fail loudly because the built .so files
121-
# # aren't present there.
122-
# rm -r src/
123-
# ls
124-
# - name: Run Python tests
125-
# run: |
126-
# pytest test -vvv
110+
- name: Delete the src/ folder just for fun
111+
run: |
112+
# The only reason we checked-out the repo is to get access to the
113+
# tests. We don't care about the rest. Out of precaution, we delete
114+
# the src/ folder to be extra sure that we're running the code from
115+
# the installed wheel rather than from the source.
116+
# This is just to be extra cautious and very overkill because a)
117+
# there's no way the `torchcodec` package from src/ can be found from
118+
# the PythonPath: the main point of `src/` is precisely to protect
119+
# against that and b) if we ever were to execute code from
120+
# `src/torchcodec`, it would fail loudly because the built .so files
121+
# aren't present there.
122+
rm -r src/
123+
ls
124+
- name: Run Python tests
125+
run: |
126+
pytest --override-ini="addopts=-v" test

0 commit comments

Comments
 (0)