Test if last passing run can be reproduced

.github/workflows/conda-build.yml

@@ -16,7 +16,7 @@ jobs:
   build:
     name: ${{ matrix.CONFIG }}
     runs-on: ${{ matrix.runs_on }}
-    timeout-minutes: 1080
+    timeout-minutes: 900
     strategy:
       fail-fast: false
       matrix:

conda-forge.yml

@@ -16,7 +16,7 @@ github:
   tooling_branch_name: main
 github_actions:
   self_hosted: true
-  timeout_minutes: 1080
+  timeout_minutes: 900
   triggers:
   - push
   - pull_request

recipe/bld.bat

@@ -7,12 +7,6 @@ if EXIST pyproject.toml (
   if %ERRORLEVEL% neq 0 exit 1
 )
 
-@REM The PyTorch test suite includes some symlinks, which aren't resolved on Windows, leading to packaging errors.
-@REM ATTN! These change and have to be updated manually, often with each release.
-@REM (no current symlinks being packaged. Leaving this information here as it took some months to find the issue. Look out
-@REM for a failure with error message: "conda_package_handling.exceptions.ArchiveCreationError: <somefile> Cannot stat
-@REM while writing file")
-
 set PYTORCH_BUILD_VERSION=%PKG_VERSION%
 @REM Always pass 0 to avoid appending ".post" to version string.
 @REM https://github.com/conda-forge/pytorch-cpu-feedstock/issues/315
@@ -27,9 +21,12 @@ if "%blas_impl%" == "generic" (
     SET BLAS=MKL
 )
 
+@REM TODO(baszalmstra): Figure out if we need these flags
+SET "USE_NUMA=0"
+SET "USE_ITT=0"
+
 if "%PKG_NAME%" == "pytorch" (
   set "PIP_ACTION=install"
-  set "PIP_VERBOSITY=-v"
   @REM We build libtorch for a specific python version.
   @REM This ensures its only build once. However, when that version changes
   @REM we need to make sure to update that here.
@@ -59,54 +56,43 @@ if "%PKG_NAME%" == "pytorch" (
   @REM For the main script we just build a wheel for so that the C++/CUDA
   @REM parts are built. Then they are reused in each python version.
   set "PIP_ACTION=wheel"
-  set "PIP_VERBOSITY=-vvv"
 )
 
-set "BUILD_CUSTOM_PROTOBUF=OFF"
-set "USE_LITE_PROTO=ON"
+if not "%cuda_compiler_version%" == "None" (
+    set USE_CUDA=1
 
-@REM TODO(baszalmstra): Figure out if we need these flags
-SET "USE_ITT=0"
-SET "USE_NUMA=0"
+    @REM set CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v%desired_cuda%
+    @REM set CUDA_BIN_PATH=%CUDA_PATH%\bin
 
-@REM TODO(baszalmstra): There are linker errors because of mixing Intel OpenMP (iomp) and Microsoft OpenMP (vcomp)
-set "USE_OPENMP=OFF"
+    set TORCH_CUDA_ARCH_LIST=5.0;6.0;6.1;7.0;7.5;8.0;8.6;8.9;9.0+PTX
 
-@REM Use our Pybind11, Eigen, sleef
-set USE_SYSTEM_EIGEN_INSTALL=1
-set USE_SYSTEM_PYBIND11=1
-set USE_SYSTEM_SLEEF=1
+    set TORCH_NVCC_FLAGS=-Xfatbin -compress-all
 
-if not "%cuda_compiler_version%" == "None" (
-    set USE_CUDA=1
     set USE_STATIC_CUDNN=0
+    set MAGMA_HOME=%PREFIX%
+
     @REM NCCL is not available on windows
     set USE_NCCL=0
     set USE_STATIC_NCCL=0
 
-    @REM set CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v%desired_cuda%
-    @REM set CUDA_BIN_PATH=%CUDA_PATH%\bin
-
-    set "TORCH_CUDA_ARCH_LIST=5.0;6.0;6.1;7.0;7.5;8.0;8.6;8.9;9.0+PTX"
-    set "TORCH_NVCC_FLAGS=-Xfatbin -compress-all"
-
     set MAGMA_HOME=%LIBRARY_PREFIX%
+
     set "PATH=%CUDA_BIN_PATH%;%PATH%"
+
     set CUDNN_INCLUDE_DIR=%LIBRARY_PREFIX%\include
-    @REM turn off very noisy nvcc warnings
-    set "CUDAFLAGS=-w --ptxas-options=-w"
+
 ) else (
     set USE_CUDA=0
-    @REM MKLDNN is an Apache-2.0 licensed library for DNNs and is used
-    @REM for CPU builds. Not to be confused with MKL.
-    set "USE_MKLDNN=1"
-
     @REM On windows, env vars are case-insensitive and setup.py
     @REM passes all env vars starting with CUDA_*, CMAKE_* to
     @REM to cmake
     set "cuda_compiler_version="
     set "cuda_compiler="
     set "CUDA_VERSION="
+
+    @REM MKLDNN is an Apache-2.0 licensed library for DNNs and is used
+    @REM for CPU builds. Not to be confused with MKL.
+    set "USE_MKLDNN=1"
 )
 
 set DISTUTILS_USE_SDK=1
@@ -130,12 +116,19 @@ set "INSTALL_TEST=0"
 set "BUILD_TEST=0"
 
 set "libuv_ROOT=%LIBRARY_PREFIX%"
+set "USE_SYSTEM_SLEEF=ON"
 
 @REM uncomment to debug cmake build
 @REM set "CMAKE_VERBOSE_MAKEFILE=1"
 
+set "BUILD_CUSTOM_PROTOBUF=OFF"
+set "USE_LITE_PROTO=ON"
+
+@REM TODO(baszalmstra): There are linker errors because of mixing Intel OpenMP (iomp) and Microsoft OpenMP (vcomp)
+set "USE_OPENMP=OFF"
+
 @REM The activation script for cuda-nvcc doesnt add the CUDA_CFLAGS on windows.
-@REM Therefore we do this manually here. See:
+@REM Therefor we do this manually here. See:
 @REM https://github.com/conda-forge/cuda-nvcc-feedstock/issues/47
 echo "CUDA_CFLAGS=%CUDA_CFLAGS%"
 set "CUDA_CFLAGS=-I%PREFIX%/Library/include -I%BUILD_PREFIX%/Library/include"
@@ -162,7 +155,7 @@ if EXIST build (
     if %ERRORLEVEL% neq 0 exit 1
 )
 
-%PYTHON% -m pip %PIP_ACTION% . --no-build-isolation --no-deps %PIP_VERBOSITY% --no-clean
+%PYTHON% -m pip %PIP_ACTION% . --no-build-isolation --no-deps -vvv --no-clean
 if %ERRORLEVEL% neq 0 exit 1
 
 @REM Here we split the build into two parts.
@@ -190,12 +183,19 @@ if "%PKG_NAME%" == "libtorch" (
     pushd torch-%PKG_VERSION%
     if %ERRORLEVEL% neq 0 exit 1
 
+    @REM Do not package `fmt.lib` (and its metadata); delete it before the move into
+    @REM %LIBRARY_BIN% because it may exist in host before installation already
+    del torch\lib\fmt.lib torch\lib\pkgconfig\fmt.pc
+    if %ERRORLEVEL% neq 0 exit 1
+    @REM also delete rest of fmt metadata
+    rmdir /s /q torch\lib\cmake\fmt
+
     @REM Move the binaries into the packages site-package directory
     @REM the only content of torch\bin, {asmjit,fbgemm}.dll, also exists in torch\lib
-    robocopy /NP /NFL /NDL /NJH /E torch\bin\ %LIBRARY_BIN%\ torch*.dll c10.dll shm.dll asmjit.dll fbgemm.dll
+    robocopy /NP /NFL /NDL /NJH /E torch\lib\ %LIBRARY_BIN%\ torch*.dll c10.dll shm.dll asmjit.dll fbgemm.dll
     robocopy /NP /NFL /NDL /NJH /E torch\lib\ %LIBRARY_LIB%\ torch*.lib c10.lib shm.lib asmjit.lib fbgemm.lib
     if not "%cuda_compiler_version%" == "None" (
-        robocopy /NP /NFL /NDL /NJH /E torch\bin\ %LIBRARY_BIN%\ c10_cuda.dll caffe2_nvrtc.dll
+        robocopy /NP /NFL /NDL /NJH /E torch\lib\ %LIBRARY_BIN%\ c10_cuda.dll caffe2_nvrtc.dll
         robocopy /NP /NFL /NDL /NJH /E torch\lib\ %LIBRARY_LIB%\ c10_cuda.lib caffe2_nvrtc.lib
     )
     robocopy /NP /NFL /NDL /NJH /E torch\share\ %LIBRARY_PREFIX%\share
@@ -216,7 +216,7 @@ if "%PKG_NAME%" == "libtorch" (
     if %ERRORLEVEL% neq 0 exit 1
 ) else if "%PKG_NAME%" == "pytorch" (
     @REM Move libtorch_python and remove the other directories afterwards.
-    robocopy /NP /NFL /NDL /NJH /E %SP_DIR%\torch\bin\ %LIBRARY_BIN%\ torch_python.dll
+    robocopy /NP /NFL /NDL /NJH /E %SP_DIR%\torch\lib\ %LIBRARY_BIN%\ torch_python.dll
     robocopy /NP /NFL /NDL /NJH /E %SP_DIR%\torch\lib\ %LIBRARY_LIB%\ torch_python.lib
     robocopy /NP /NFL /NDL /NJH /E %SP_DIR%\torch\lib\ %LIBRARY_LIB%\ _C.lib
     rmdir /s /q %SP_DIR%\torch\lib

recipe/build.sh

@@ -1,10 +1,8 @@
 #!/bin/bash
 
-set -ex
+echo "=== Building ${PKG_NAME} (py: ${PY_VER}) ==="
 
-echo "#########################################################################"
-echo "Building ${PKG_NAME} (py: ${PY_VER}) using BLAS implementation $blas_impl"
-echo "#########################################################################"
+set -ex
 
 # This is used to detect if it's in the process of building pytorch
 export IN_PYTORCH_BUILD=1
@@ -22,22 +20,9 @@ rm -rf pyproject.toml
 export USE_CUFILE=0
 export USE_NUMA=0
 export USE_ITT=0
-
-#################### ADJUST COMPILER AND LINKER FLAGS #####################
-# Pytorch's build system doesn't like us setting the c++ standard through CMAKE_CXX_FLAGS
-# and will issue a warning.  We need to use at least C++17 to match the abseil ABI, see
-# https://github.com/conda-forge/abseil-cpp-feedstock/issues/45, which pytorch 2.5 uses already:
-# https://github.com/pytorch/pytorch/blob/v2.5.1/CMakeLists.txt#L36-L48
-export CXXFLAGS="$(echo $CXXFLAGS | sed 's/-std=c++[0-9][0-9]//g')"
-# The below three lines expose symbols that would otherwise be hidden or
-# optimised away. They were here before, so removing them would potentially
-# break users' programs
 export CFLAGS="$(echo $CFLAGS | sed 's/-fvisibility-inlines-hidden//g')"
 export CXXFLAGS="$(echo $CXXFLAGS | sed 's/-fvisibility-inlines-hidden//g')"
 export LDFLAGS="$(echo $LDFLAGS | sed 's/-Wl,--as-needed//g')"
-# The default conda LDFLAGs include -Wl,-dead_strip_dylibs, which removes all the
-# MKL sequential, core, etc. libraries, resulting in a "Symbol not found: _mkl_blas_caxpy"
-# error on osx-64.
 export LDFLAGS="$(echo $LDFLAGS | sed 's/-Wl,-dead_strip_dylibs//g')"
 export LDFLAGS_LD="$(echo $LDFLAGS_LD | sed 's/-dead_strip_dylibs//g')"
 if [[ "$c_compiler" == "clang" ]]; then
@@ -60,7 +45,6 @@ fi
 # can be imported on system without a GPU
 LDFLAGS="${LDFLAGS//-Wl,-z,now/-Wl,-z,lazy}"
 
-################ CONFIGURE CMAKE FOR CONDA ENVIRONMENT ###################
 export CMAKE_GENERATOR=Ninja
 export CMAKE_LIBRARY_PATH=$PREFIX/lib:$PREFIX/include:$CMAKE_LIBRARY_PATH
 export CMAKE_PREFIX_PATH=$PREFIX
@@ -89,8 +73,6 @@ export USE_SYSTEM_SLEEF=1
 # use our protobuf
 export BUILD_CUSTOM_PROTOBUF=OFF
 rm -rf $PREFIX/bin/protoc
-export USE_SYSTEM_PYBIND11=1
-export USE_SYSTEM_EIGEN_INSTALL=1
 
 # prevent six from being downloaded
 > third_party/NNPACK/cmake/DownloadSix.cmake
@@ -116,28 +98,17 @@ if [[ "${CI}" == "github_actions" ]]; then
     # reduce parallelism to avoid getting OOM-killed on
     # cirun-openstack-gpu-2xlarge, which has 32GB RAM, 8 CPUs
     export MAX_JOBS=4
-elif [[ "${CI}" == "azure" ]]; then
-    export MAX_JOBS=${CPU_COUNT}
 else
-    # Leave a spare core for other tasks, per common practice.
-    # Reducing further can help with out-of-memory errors.
-    export MAX_JOBS=$((CPU_COUNT > 1 ? CPU_COUNT - 1 : 1))
+    export MAX_JOBS=${CPU_COUNT}
 fi
 
-case "$blas_impl" in
-    "generic")
-        # Fake openblas
-        export BLAS=OpenBLAS
-        export OpenBLAS_HOME=${PREFIX}
-        ;;
-    "mkl")
-        export BLAS=MKL
-        ;;
-    *)
-        echo "[ERROR] Unsupported BLAS implementation '${blas_impl}'" >&2
-        exit 1
-        ;;
-esac
+if [[ "$blas_impl" == "generic" ]]; then
+    # Fake openblas
+    export BLAS=OpenBLAS
+    export OpenBLAS_HOME=${PREFIX}
+else
+    export BLAS=MKL
+fi
 
 if [[ "$PKG_NAME" == "pytorch" ]]; then
   # Trick Cmake into thinking python hasn't changed
@@ -176,9 +147,11 @@ elif [[ ${cuda_compiler_version} != "None" ]]; then
     # all of them.
     export CUDAToolkit_BIN_DIR=${BUILD_PREFIX}/bin
     export CUDAToolkit_ROOT_DIR=${PREFIX}
+    if [[ "${target_platform}" != "${build_platform}" ]]; then
+        export CUDA_TOOLKIT_ROOT=${PREFIX}
+    fi
     # for CUPTI
     export CUDA_TOOLKIT_ROOT_DIR=${PREFIX}
-    export CUDAToolkit_ROOT=${PREFIX}
     case ${target_platform} in
         linux-64)
             export CUDAToolkit_TARGET_DIR=${PREFIX}/targets/x86_64-linux
@@ -190,24 +163,12 @@ elif [[ ${cuda_compiler_version} != "None" ]]; then
             echo "unknown CUDA arch, edit build.sh"
             exit 1
     esac
-
-    # Compatibility matrix for update: https://en.wikipedia.org/wiki/CUDA#GPUs_supported
-    # Warning from pytorch v1.12.1: In the future we will require one to
-    # explicitly pass TORCH_CUDA_ARCH_LIST to cmake instead of implicitly
-    # setting it as an env variable.
-    # Doing this is nontrivial given that we're using setup.py as an entry point, but should
-    # be addressed to pre-empt upstream changing it, as it probably won't result in a failed
-    # configuration.
-    #
-    # See:
-    # https://pytorch.org/docs/stable/cpp_extension.html (Compute capabilities)
-    # https://github.com/pytorch/pytorch/blob/main/.ci/manywheel/build_cuda.sh
     case ${cuda_compiler_version} in
-        12.[0-6])
+        12.6)
             export TORCH_CUDA_ARCH_LIST="5.0;6.0;6.1;7.0;7.5;8.0;8.6;8.9;9.0+PTX"
             ;;
         *)
-            echo "No CUDA architecture list exists for CUDA v${cuda_compiler_version}. See build.sh for information on adding one."
+            echo "unsupported cuda version. edit build.sh"
             exit 1
     esac
     export TORCH_NVCC_FLAGS="-Xfatbin -compress-all"
@@ -219,8 +180,6 @@ elif [[ ${cuda_compiler_version} != "None" ]]; then
     export USE_STATIC_CUDNN=0
     export MAGMA_HOME="${PREFIX}"
     export USE_MAGMA=1
-    # turn off noisy nvcc warnings
-    export CMAKE_CUDA_FLAGS="-w --ptxas-options=-w"
 else
     if [[ "$target_platform" != *-64 ]]; then
       # Breakpad seems to not work on aarch64 or ppc64le
@@ -244,16 +203,15 @@ case ${PKG_NAME} in
 
     mv build/lib.*/torch/bin/* ${PREFIX}/bin/
     mv build/lib.*/torch/lib/* ${PREFIX}/lib/
-    # need to merge these now because we're using system pybind11, meaning the destination directory is not empty
-    rsync -a build/lib.*/torch/share/* ${PREFIX}/share/
+    mv build/lib.*/torch/share/* ${PREFIX}/share/
     mv build/lib.*/torch/include/{ATen,caffe2,tensorpipe,torch,c10} ${PREFIX}/include/
     rm ${PREFIX}/lib/libtorch_python.*
 
     # Keep the original backed up to sed later
     cp build/CMakeCache.txt build/CMakeCache.txt.orig
     ;;
   pytorch)
-    $PREFIX/bin/python -m pip install . --no-deps --no-build-isolation -v --no-clean \
+    $PREFIX/bin/python -m pip install . --no-deps -vvv --no-clean \
         | sed "s,${CXX},\$\{CXX\},g" \
         | sed "s,${PREFIX},\$\{PREFIX\},g"
     # Keep this in ${PREFIX}/lib so that the library can be found by