From e034b15e736a6be2bca57281f10c0d16196f4d1f Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Tue, 2 Dec 2025 13:26:58 +0000 Subject: [PATCH 1/7] Add aarch64 ffmpeg targets --- ...fetch_and_expose_non_gpl_ffmpeg_libs.cmake | 76 +++++++++++++------ 1 file changed, 52 insertions(+), 24 deletions(-) diff --git a/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake b/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake index 1200499dd..b2f5cb00f 100644 --- a/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake +++ b/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake @@ -15,31 +15,59 @@ set( ) if (LINUX) - set( - platform_url - ${base_url}/linux_x86_64 - ) + if (CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64|ARM64") + set( + platform_url + ${base_url}/linux_aarch64 + ) - set( - f4_sha256 - 1a083f1922443bedb5243d04896383b8c606778a7ddb9d886c8303e55339fe0c - ) - set( - f5_sha256 - 65d6ad54082d94dcb3f801d73df2265e0e1bb303c7afbce7723e3b77ccd0e207 - ) - set( - f6_sha256 - 8bd5939c2f4a4b072e837e7870c13fe7d13824e5ff087ab534e4db4e90b7be9c - ) - set( - f7_sha256 - 1cb946d8b7c6393c2c3ebe1f900b8de7a2885fe614c45d4ec32c9833084f2f26 - ) - set( - f8_sha256 - c55b3c1a4b5e4d5fdd7c632bea3ab6f45b4e37cc8e0999dda3f84a8ed8defad8 - ) + set( + f4_sha256 + a310a2ed9ffe555fd3278dae15065541098dd35e124564671dcda6a6620ac842 + ) + set( + f5_sha256 + 89ca7996bccbc2db49adaa401d20fdbabffe0e1b4e07a0f81d6b143e858b7c8d + ) + set( + f6_sha256 + ae44c67b4587d061b8e9cc8990ca891ee013fe52ad79e5016ba29871562621da + ) + set( + f7_sha256 + 948e2cac66ca6f68ff526d5e84138e94bce0f1a7c83f502d15d85d0bd3ddc112 + ) + set( + f8_sha256 + b9cfd99ae75a14e58300854967d4dc49de0b3daa551df51ea1f52a3f08d2c8af + ) + elseif (LINUX) # assume x86_64 + set( + platform_url + ${base_url}/linux_x86_64 + ) + + set( + f4_sha256 + 1a083f1922443bedb5243d04896383b8c606778a7ddb9d886c8303e55339fe0c + ) + set( + f5_sha256 + 65d6ad54082d94dcb3f801d73df2265e0e1bb303c7afbce7723e3b77ccd0e207 + ) + set( + f6_sha256 + 8bd5939c2f4a4b072e837e7870c13fe7d13824e5ff087ab534e4db4e90b7be9c + ) + set( + f7_sha256 + 1cb946d8b7c6393c2c3ebe1f900b8de7a2885fe614c45d4ec32c9833084f2f26 + ) + set( + f8_sha256 + c55b3c1a4b5e4d5fdd7c632bea3ab6f45b4e37cc8e0999dda3f84a8ed8defad8 + ) + endif() elseif (APPLE) set( platform_url From f07d5ffaee8c6d8b897b7b87e173b22ef4438dd5 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Tue, 2 Dec 2025 13:31:50 +0000 Subject: [PATCH 2/7] Add aarch64 wheel build CI job --- .../workflows/linux_cuda_aarch64_wheel.yaml | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 .github/workflows/linux_cuda_aarch64_wheel.yaml diff --git a/.github/workflows/linux_cuda_aarch64_wheel.yaml b/.github/workflows/linux_cuda_aarch64_wheel.yaml new file mode 100644 index 000000000..3be5539b0 --- /dev/null +++ b/.github/workflows/linux_cuda_aarch64_wheel.yaml @@ -0,0 +1,58 @@ +name: Build and test Linux CUDA aarch64 wheels + +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + tags: + - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ github.ref_type == 'branch' && github.sha }}-${{ github.event_name == 'workflow_dispatch' }} + cancel-in-progress: true + +permissions: + id-token: write + contents: write + +defaults: + run: + shell: bash -l -eo pipefail {0} + +jobs: + generate-matrix: + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + with: + package-type: wheel + os: linux-aarch64 + test-infra-repository: pytorch/test-infra + test-infra-ref: main + with-cpu: disable + with-xpu: disable + with-rocm: disable + with-cuda: enable + build-python-only: "disable" + build: + needs: generate-matrix + strategy: + fail-fast: false + name: Build and Upload wheel + uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main + with: + repository: meta-pytorch/torchcodec + ref: "" + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.generate-matrix.outputs.matrix }} + pre-script: packaging/pre_build_script.sh + post-script: packaging/post_build_script.sh + smoke-test-script: packaging/fake_smoke_test.py + package-name: torchcodec + trigger-event: ${{ github.event_name }} + architecture: aarch64 + build-platform: "python-build-package" + build-command: "BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 ENABLE_CUDA=1 python -m build --wheel -vvv --no-isolation" From 89a544435c59d332c1d0a8a794c8a0f97beadaf8 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Tue, 2 Dec 2025 13:32:51 +0000 Subject: [PATCH 3/7] Temporarily disable other jobs --- .github/workflows/cpp_tests.yaml | 2 +- .github/workflows/docs.yaml | 2 +- .github/workflows/lint.yaml | 2 +- .github/workflows/linux_cuda_wheel.yaml | 2 +- .github/workflows/linux_wheel.yaml | 2 +- .github/workflows/macos_wheel.yaml | 2 +- .github/workflows/windows_wheel.yaml | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/cpp_tests.yaml b/.github/workflows/cpp_tests.yaml index e08d90754..a3e81f768 100644 --- a/.github/workflows/cpp_tests.yaml +++ b/.github/workflows/cpp_tests.yaml @@ -1,7 +1,7 @@ name: CPP tests on: - push: + # push: branches: [ main ] pull_request: diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 92daeddbd..c86571784 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -3,7 +3,7 @@ name: Docs on: push: branches: [ main ] - pull_request: + # pull_request: permissions: id-token: write diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 57a4916a4..de9aa75a0 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -3,7 +3,7 @@ name: Lint on: push: branches: [ main ] - pull_request: + # pull_request: concurrency: group: unit-test${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_number || github.ref }} diff --git a/.github/workflows/linux_cuda_wheel.yaml b/.github/workflows/linux_cuda_wheel.yaml index 4e711b8c3..15a944703 100644 --- a/.github/workflows/linux_cuda_wheel.yaml +++ b/.github/workflows/linux_cuda_wheel.yaml @@ -1,7 +1,7 @@ name: Build and test Linux CUDA wheels on: - pull_request: + # pull_request: push: branches: - nightly diff --git a/.github/workflows/linux_wheel.yaml b/.github/workflows/linux_wheel.yaml index cccbedc25..d28da8c68 100644 --- a/.github/workflows/linux_wheel.yaml +++ b/.github/workflows/linux_wheel.yaml @@ -1,7 +1,7 @@ name: Build and test Linux wheel on: - pull_request: + # pull_request: push: branches: - nightly diff --git a/.github/workflows/macos_wheel.yaml b/.github/workflows/macos_wheel.yaml index ead45784d..3564824de 100644 --- a/.github/workflows/macos_wheel.yaml +++ b/.github/workflows/macos_wheel.yaml @@ -1,7 +1,7 @@ name: Build and test MacOS wheel on: - pull_request: + # pull_request: push: branches: - nightly diff --git a/.github/workflows/windows_wheel.yaml b/.github/workflows/windows_wheel.yaml index 8a9b5b740..968b59984 100644 --- a/.github/workflows/windows_wheel.yaml +++ b/.github/workflows/windows_wheel.yaml @@ -1,7 +1,7 @@ name: Build and test Windows wheel on: - pull_request: + # pull_request: push: branches: - nightly From c9b3df61b7b67a2473dfa899c5b27f18af2f6651 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Tue, 2 Dec 2025 15:01:19 +0000 Subject: [PATCH 4/7] Add test job? --- .../workflows/linux_cuda_aarch64_wheel.yaml | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/.github/workflows/linux_cuda_aarch64_wheel.yaml b/.github/workflows/linux_cuda_aarch64_wheel.yaml index 3be5539b0..3ab00ce49 100644 --- a/.github/workflows/linux_cuda_aarch64_wheel.yaml +++ b/.github/workflows/linux_cuda_aarch64_wheel.yaml @@ -56,3 +56,93 @@ jobs: architecture: aarch64 build-platform: "python-build-package" build-command: "BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 ENABLE_CUDA=1 python -m build --wheel -vvv --no-isolation" + + install-and-test: + runs-on: linux.arm64.2xlarge + strategy: + fail-fast: false + matrix: + # 3.10 corresponds to the minimum python version for which we build + # the wheel unless the label cliflow/binaries/all is present in the + # PR. + # For the actual release we should add that label and change this to + # include more python versions. + python-version: ['3.10'] + # We test against 12.6 and 13.0 to avoid having too big of a CI matrix, + # but for releases we should add 12.8. + cuda-version: ['12.6', '13.0'] + # TODO: put back ffmpeg 5 https://github.com/pytorch/torchcodec/issues/325 + ffmpeg-version-for-tests: ['4.4.2', '6', '7', '8.0'] + + container: + image: "pytorch/manylinux2_28-builder:cuda${{ matrix.cuda-version }}" + options: "--gpus all -e NVIDIA_DRIVER_CAPABILITIES=video,compute,utility" + needs: build + steps: + - name: Setup env vars + run: | + cuda_version_without_periods=$(echo "${{ matrix.cuda-version }}" | sed 's/\.//g') + echo cuda_version_without_periods=${cuda_version_without_periods} >> $GITHUB_ENV + python_version_without_periods=$(echo "${{ matrix.python-version }}" | sed 's/\.//g') + echo python_version_without_periods=${python_version_without_periods} >> $GITHUB_ENV + - uses: actions/download-artifact@v4 + with: + name: meta-pytorch_torchcodec__${{ matrix.python-version }}_cu${{ env.cuda_version_without_periods }}_aarch64 + path: pytorch/torchcodec/dist/ + - name: Setup miniconda using test-infra + uses: pytorch/test-infra/.github/actions/setup-miniconda@main + with: + python-version: ${{ matrix.python-version }} + # We install conda packages at the start because otherwise conda may have conflicts with dependencies. + # Note: xorg-libxau was addded to fix a problem with ffmpeg 4. We should consider removing it. + default-packages: "nvidia/label/cuda-${{ matrix.cuda-version }}.0::libnpp nvidia::cuda-nvrtc=${{ matrix.cuda-version }} nvidia::cuda-toolkit=${{ matrix.cuda-version }} nvidia::cuda-cudart=${{ matrix.cuda-version }} nvidia::cuda-driver-dev=${{ matrix.cuda-version }} conda-forge::ffmpeg=${{ matrix.ffmpeg-version-for-tests }} conda-forge::xorg-libxau" + - name: Check env, set LD_LIBRARY_PATH + run: | + ${CONDA_RUN} env + ${CONDA_RUN} conda info + ${CONDA_RUN} nvidia-smi + ${CONDA_RUN} conda list + echo LD_LIBRARY_PATH=$CONDA_PREFIX/lib:/usr/local/cuda/lib64/:${LD_LIBRARY_PATH} >> $GITHUB_ENV + - name: Assert ffmpeg exists + run: | + ${CONDA_RUN} ffmpeg -buildconf + - name: Update pip + run: ${CONDA_RUN} python -m pip install --upgrade pip + - name: Install PyTorch + run: | + ${CONDA_RUN} python -m pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cu${{ env.cuda_version_without_periods }} + ${CONDA_RUN} python -c 'import torch; print(f"{torch.__version__}"); print(f"{torch.__file__}"); print(f"{torch.cuda.is_available()=}")' + - name: Install torchcodec from the wheel + run: | + wheel_path=`find pytorch/torchcodec/dist -type f -name "*cu${{ env.cuda_version_without_periods }}-cp${{ env.python_version_without_periods }}*.whl"` + echo Installing $wheel_path + ${CONDA_RUN} python -m pip install $wheel_path -vvv + + - name: Check out repo + uses: actions/checkout@v3 + + - name: Install test dependencies + run: | + # Ideally we would find a way to get those dependencies from pyproject.toml + ${CONDA_RUN} python -m pip install numpy pytest pillow + + - name: Delete the src/ folder just for fun + run: | + # The only reason we checked-out the repo is to get access to the + # tests. We don't care about the rest. Out of precaution, we delete + # the src/ folder to be extra sure that we're running the code from + # the installed wheel rather than from the source. + # This is just to be extra cautious and very overkill because a) + # there's no way the `torchcodec` package from src/ can be found from + # the PythonPath: the main point of `src/` is precisely to protect + # against that and b) if we ever were to execute code from + # `src/torchcodec`, it would fail loudly because the built .so files + # aren't present there. + rm -r src/ + ls + - name: Run Python tests + run: | + ${CONDA_RUN} FAIL_WITHOUT_CUDA=1 pytest --override-ini="addopts=-v" test --tb=short + - name: Run Python benchmark + run: | + ${CONDA_RUN} time python benchmarks/decoders/gpu_benchmark.py --devices=cuda:0,cpu --resize_devices=none From ae3824d81e031eb62d7063ef5dbd19e55fdde271 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Tue, 2 Dec 2025 18:36:00 +0000 Subject: [PATCH 5/7] Try testing on CPU then --- .../workflows/linux_cuda_aarch64_wheel.yaml | 77 ++++++++----------- 1 file changed, 32 insertions(+), 45 deletions(-) diff --git a/.github/workflows/linux_cuda_aarch64_wheel.yaml b/.github/workflows/linux_cuda_aarch64_wheel.yaml index 3ab00ce49..163b1e8c3 100644 --- a/.github/workflows/linux_cuda_aarch64_wheel.yaml +++ b/.github/workflows/linux_cuda_aarch64_wheel.yaml @@ -59,72 +59,61 @@ jobs: install-and-test: runs-on: linux.arm64.2xlarge + container: + image: pytorch/manylinuxaarch64-builder:cuda12.6 + env: + cuda_version_without_periods: "126" strategy: fail-fast: false matrix: - # 3.10 corresponds to the minimum python version for which we build - # the wheel unless the label cliflow/binaries/all is present in the - # PR. - # For the actual release we should add that label and change this to - # include more python versions. python-version: ['3.10'] - # We test against 12.6 and 13.0 to avoid having too big of a CI matrix, - # but for releases we should add 12.8. - cuda-version: ['12.6', '13.0'] - # TODO: put back ffmpeg 5 https://github.com/pytorch/torchcodec/issues/325 - ffmpeg-version-for-tests: ['4.4.2', '6', '7', '8.0'] - - container: - image: "pytorch/manylinux2_28-builder:cuda${{ matrix.cuda-version }}" - options: "--gpus all -e NVIDIA_DRIVER_CAPABILITIES=video,compute,utility" + ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1', '8.0'] needs: build steps: - - name: Setup env vars - run: | - cuda_version_without_periods=$(echo "${{ matrix.cuda-version }}" | sed 's/\.//g') - echo cuda_version_without_periods=${cuda_version_without_periods} >> $GITHUB_ENV - python_version_without_periods=$(echo "${{ matrix.python-version }}" | sed 's/\.//g') - echo python_version_without_periods=${python_version_without_periods} >> $GITHUB_ENV - uses: actions/download-artifact@v4 with: name: meta-pytorch_torchcodec__${{ matrix.python-version }}_cu${{ env.cuda_version_without_periods }}_aarch64 path: pytorch/torchcodec/dist/ - - name: Setup miniconda using test-infra - uses: pytorch/test-infra/.github/actions/setup-miniconda@main + - name: Setup conda env + uses: conda-incubator/setup-miniconda@v3 with: + auto-update-conda: true + # Using miniforge instead of miniconda ensures that the default + # conda channel is conda-forge instead of main/default. This ensures + # ABI consistency between dependencies: + # https://conda-forge.org/docs/user/transitioning_from_defaults/ + miniforge-version: latest + activate-environment: test python-version: ${{ matrix.python-version }} - # We install conda packages at the start because otherwise conda may have conflicts with dependencies. - # Note: xorg-libxau was addded to fix a problem with ffmpeg 4. We should consider removing it. - default-packages: "nvidia/label/cuda-${{ matrix.cuda-version }}.0::libnpp nvidia::cuda-nvrtc=${{ matrix.cuda-version }} nvidia::cuda-toolkit=${{ matrix.cuda-version }} nvidia::cuda-cudart=${{ matrix.cuda-version }} nvidia::cuda-driver-dev=${{ matrix.cuda-version }} conda-forge::ffmpeg=${{ matrix.ffmpeg-version-for-tests }} conda-forge::xorg-libxau" - - name: Check env, set LD_LIBRARY_PATH - run: | - ${CONDA_RUN} env - ${CONDA_RUN} conda info - ${CONDA_RUN} nvidia-smi - ${CONDA_RUN} conda list - echo LD_LIBRARY_PATH=$CONDA_PREFIX/lib:/usr/local/cuda/lib64/:${LD_LIBRARY_PATH} >> $GITHUB_ENV - - name: Assert ffmpeg exists - run: | - ${CONDA_RUN} ffmpeg -buildconf - name: Update pip - run: ${CONDA_RUN} python -m pip install --upgrade pip + run: python -m pip install --upgrade pip - name: Install PyTorch run: | - ${CONDA_RUN} python -m pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cu${{ env.cuda_version_without_periods }} - ${CONDA_RUN} python -c 'import torch; print(f"{torch.__version__}"); print(f"{torch.__file__}"); print(f"{torch.cuda.is_available()=}")' + python -m pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cpu - name: Install torchcodec from the wheel run: | - wheel_path=`find pytorch/torchcodec/dist -type f -name "*cu${{ env.cuda_version_without_periods }}-cp${{ env.python_version_without_periods }}*.whl"` + wheel_path=`find pytorch/torchcodec/dist -type f -name "*.whl"` echo Installing $wheel_path - ${CONDA_RUN} python -m pip install $wheel_path -vvv + python -m pip install $wheel_path -vvv - name: Check out repo uses: actions/checkout@v3 + - name: Install ffmpeg, post build + run: | + # Ideally we would have checked for that before installing the wheel, + # but we need to checkout the repo to access this file, and we don't + # want to checkout the repo before installing the wheel to avoid any + # side-effect. It's OK. + source packaging/helpers.sh + assert_ffmpeg_not_installed + + conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge + ffmpeg -version - name: Install test dependencies run: | # Ideally we would find a way to get those dependencies from pyproject.toml - ${CONDA_RUN} python -m pip install numpy pytest pillow + python -m pip install numpy pytest pillow - name: Delete the src/ folder just for fun run: | @@ -142,7 +131,5 @@ jobs: ls - name: Run Python tests run: | - ${CONDA_RUN} FAIL_WITHOUT_CUDA=1 pytest --override-ini="addopts=-v" test --tb=short - - name: Run Python benchmark - run: | - ${CONDA_RUN} time python benchmarks/decoders/gpu_benchmark.py --devices=cuda:0,cpu --resize_devices=none + pytest --override-ini="addopts=-v" test + From 24600631e10ba420693b4db23e24efa4b0283b53 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Wed, 3 Dec 2025 09:47:34 +0000 Subject: [PATCH 6/7] Try without ENABLE_CUDA? --- .github/workflows/linux_cuda_aarch64_wheel.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux_cuda_aarch64_wheel.yaml b/.github/workflows/linux_cuda_aarch64_wheel.yaml index 163b1e8c3..23ed2feb0 100644 --- a/.github/workflows/linux_cuda_aarch64_wheel.yaml +++ b/.github/workflows/linux_cuda_aarch64_wheel.yaml @@ -55,7 +55,7 @@ jobs: trigger-event: ${{ github.event_name }} architecture: aarch64 build-platform: "python-build-package" - build-command: "BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 ENABLE_CUDA=1 python -m build --wheel -vvv --no-isolation" + build-command: "BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 python -m build --wheel -vvv --no-isolation" install-and-test: runs-on: linux.arm64.2xlarge From aa6a9bedfde966f24bb7997f43357dd7ce7d1c0a Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Wed, 3 Dec 2025 10:28:48 +0000 Subject: [PATCH 7/7] Add LD_LIBRARY_PATH --- .github/workflows/linux_cuda_aarch64_wheel.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/linux_cuda_aarch64_wheel.yaml b/.github/workflows/linux_cuda_aarch64_wheel.yaml index 23ed2feb0..c100c5dda 100644 --- a/.github/workflows/linux_cuda_aarch64_wheel.yaml +++ b/.github/workflows/linux_cuda_aarch64_wheel.yaml @@ -109,6 +109,7 @@ jobs: conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge ffmpeg -version + echo LD_LIBRARY_PATH=$CONDA_PREFIX/lib:/usr/local/cuda/lib64/:${LD_LIBRARY_PATH} >> $GITHUB_ENV - name: Install test dependencies run: | @@ -131,5 +132,6 @@ jobs: ls - name: Run Python tests run: | + pytest --override-ini="addopts=-v" test