chore: replace deprecated windows ci container (PolusAI#283)

sameeul · web-flow · commit ad6205501844 · 2025-07-16T15:02:06.000-04:00
diff --git a/.github/workflows/build_cuda11_wheels.yml b/.github/workflows/build_cuda11_wheels.yml
@@ -8,9 +8,11 @@ jobs:
   build_wheels:
     name: Build wheels on ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
+    continue-on-error: true
     strategy:
+      fail-fast: false
       matrix:
-        os: [windows-2019, ubuntu-22.04]
+        os: [ubuntu-22.04]
         cibw_archs: ["auto64"]
         cibw_build: ["cp39", "cp310", "cp311", "cp312"]
 
@@ -20,26 +22,6 @@ jobs:
         with:
           submodules: recursive    
 
-      - uses: ilammy/msvc-dev-cmd@v1
-        name: Add MSVS Path
-
-      - name: Install CUDA
-        if: runner.os == 'Windows'
-        env: 
-          cuda: "11.8.0"
-          visual_studio: "Visual Studio 16 2019"
-        shell: powershell
-        run: .\ci-utils\install_cuda_windows.ps1
-
-      - name: nvcc check
-        if: runner.os == 'Windows'
-        shell: powershell
-        run: |
-          nvcc -V
-          ls $env:CUDA_PATH
-          ls $env:CUDA_PATH\bin
-          ls $env:CUDA_PATH\include
-
       - uses: actions/setup-python@v4
         name: Install Python
         with:
@@ -49,9 +31,10 @@ jobs:
         run: |
           python -m pip install cibuildwheel==2.16.2 delvewheel wheel
 
-      - name: Building wheels 
+
+      - name: Building wheels on Linux
         run: |
-          python -m cibuildwheel --output-dir dist
+              python -m cibuildwheel --output-dir dist
         env:
           CIBW_BUILD: ${{ matrix.cibw_build }}-*
           CIBW_SKIP: "*musllinux*"
@@ -65,16 +48,13 @@ jobs:
                                    bash ci-utils/install_prereq_linux.sh --build_arrow yes &&
                                    mkdir -p /tmp/nyxus_bld &&
                                    cp -r local_install /tmp/nyxus_bld
-          CIBW_BEFORE_ALL_WINDOWS: nvcc -V &&
-                                   ci-utils\install_prereq_win.bat &&
-                                   xcopy /E /I /y local_install C:\TEMP\nyxus_bld\local_install
           CIBW_ENVIRONMENT_LINUX: LD_LIBRARY_PATH="/tmp/nyxus_bld/local_install/lib:/tmp/nyxus_bld/local_install/lib64:/usr/local/cuda/targets/x86_64-linux/lib:$LD_LIBRARY_PATH" CPATH="/usr/local/cuda/targets/x86_64-linux/include:$CPATH" PATH="/usr/local/cuda/bin:$PATH" NYXUS_DEP_DIR="/tmp/nyxus_bld/local_install" CXXFLAGS="-I /usr/local/cuda/include" CMAKE_ARGS="-DUSEGPU=ON -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc -DCMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES=/usr/local/cuda/include" NYXUS_GPU_WHEEL="ON" 
-          CIBW_ENVIRONMENT_WINDOWS: PATH="C:\\TEMP\\nyxus_bld\\local_install\\bin;$PATH" NYXUS_DEP_DIR="C:\\TEMP\\nyxus_bld\\local_install" CMAKE_ARGS="-DUSEGPU=ON" NYXUS_GPU_WHEEL="ON" 
           CIBW_REPAIR_WHEEL_COMMAND_LINUX: "auditwheel repair --exclude=libcufft.so --exclude=libcufft.so.10 --exclude=libcufft.so.10.4.2.109 --exclude=libcudart.so --exclude=libcudart.so.11.0 --exclude=libcudart.so.11.3.109  -w {dest_dir} {wheel}"
-          CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair -vv -w {dest_dir} {wheel} --no-dll=cufft64_10.dll;cufftw64_10.dll;cudart32_110.dll;cudart64_110.dll"
           CIBW_ARCHS: ${{ matrix.cibw_archs }}
           CIBW_TEST_REQUIRES: numpy pandas pyarrow pytest bfio
-          CIBW_BEFORE_TEST_WINDOWS: xcopy /E /I /y "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8\bin" %SystemRoot%\System32
+          CIBW_BEFORE_TEST_LINUX: >-
+            pip install "numpy<2.3.1" --only-binary=:all:
+          CIBW_TEST_REQUIRES_LINUX: pandas pyarrow pytest bfio
           CIBW_TEST_COMMAND: python -m pip check
 
       - name: Upload Artifact
diff --git a/.github/workflows/build_cuda12_wheels.yml b/.github/workflows/build_cuda12_wheels.yml
@@ -8,9 +8,11 @@ jobs:
   build_wheels:
     name: Build wheels on ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
+    continue-on-error: true
     strategy:
+      fail-fast: false
       matrix:
-        os: [windows-2019, ubuntu-22.04]
+        os: [windows-2022, ubuntu-22.04]
         cibw_archs: ["auto64"]
         cibw_build: ["cp39", "cp310", "cp311", "cp312"]
 
@@ -26,8 +28,8 @@ jobs:
       - name: Install CUDA
         if: runner.os == 'Windows'
         env: 
-          cuda: "12.0.0"
-          visual_studio: "Visual Studio 16 2019"
+          cuda: "12.4.0"
+          visual_studio: "Visual Studio 17 2022"
         shell: powershell
         run: .\ci-utils\install_cuda_windows.ps1
 
@@ -56,11 +58,8 @@ jobs:
           CIBW_BUILD: ${{ matrix.cibw_build }}-*
           CIBW_SKIP: "*musllinux*"
           CIBW_BUILD_VERBOSITY: 3
-          CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
-          CIBW_BEFORE_ALL_LINUX: sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo &&
-                                   sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo &&
-                                   sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo && 
-                                   yum install -y llvm libevent-devel openssl-devel && 
+          CIBW_MANYLINUX_X86_64_IMAGE: quay.io/pypa/manylinux_2_28_x86_64:latest
+          CIBW_BEFORE_ALL_LINUX: dnf -y install llvm libevent-devel openssl-devel && 
                                    bash ci-utils/install_cuda_yum.sh 12 && 
                                    bash ci-utils/install_prereq_linux.sh --build_arrow yes &&
                                    mkdir -p /tmp/nyxus_bld &&
@@ -74,7 +73,7 @@ jobs:
           CIBW_REPAIR_WHEEL_COMMAND_LINUX: auditwheel repair --exclude=libcufft.so --exclude=libcufft.so.11 --exclude=libcufft.so.11.0.12.1  --exclude=libcudart.so --exclude=libcudart.so.12 --exclude=libcudart.so.12.3.101  -w {dest_dir} {wheel}
           CIBW_ARCHS: ${{ matrix.cibw_archs }}
           CIBW_TEST_REQUIRES: numpy pandas pyarrow pytest bfio
-          CIBW_BEFORE_TEST_WINDOWS: xcopy /E /I /y "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.0\bin" %SystemRoot%\System32
+          CIBW_BEFORE_TEST_WINDOWS: xcopy /E /I /y "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\bin" %SystemRoot%\System32
           CIBW_TEST_COMMAND: python -m pip check
 
       - name: Upload Artifact
diff --git a/.github/workflows/publish_cuda11_pypi.yml b/.github/workflows/publish_cuda11_pypi.yml
@@ -12,7 +12,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [windows-2019, ubuntu-22.04]
+        os: [windows-2022, ubuntu-22.04]
         cibw_archs: ["auto64"]
         cibw_build: ["cp39-*", "cp310-*", "cp311-*", "cp312-*"]
 
@@ -22,26 +22,6 @@ jobs:
         with:
           submodules: recursive    
 
-      - uses: ilammy/msvc-dev-cmd@v1
-        name: Add MSVS Path
-
-      - name: Install CUDA
-        if: runner.os == 'Windows'
-        env: 
-          cuda: "11.8.0"
-          visual_studio: "Visual Studio 16 2019"
-        shell: powershell
-        run: .\ci-utils\install_cuda_windows.ps1
-
-      - name: nvcc check
-        if: runner.os == 'Windows'
-        shell: powershell
-        run: |
-          nvcc -V
-          ls $env:CUDA_PATH
-          ls $env:CUDA_PATH\bin
-          ls $env:CUDA_PATH\include
-
       - uses: actions/setup-python@v4
         name: Install Python
         with:
@@ -51,9 +31,9 @@ jobs:
         run: |
           python -m pip install cibuildwheel==2.16.2 delvewheel wheel
 
-      - name: Building wheels 
+      - name: Building wheels on Linux
         run: |
-          python -m cibuildwheel --output-dir dist
+              python -m cibuildwheel --output-dir dist
         env:
           CIBW_BUILD: ${{ matrix.cibw_build }}
           CIBW_SKIP: "*musllinux*"
@@ -67,18 +47,16 @@ jobs:
                                    bash ci-utils/install_prereq_linux.sh --build_arrow yes &&
                                    mkdir -p /tmp/nyxus_bld &&
                                    cp -r local_install /tmp/nyxus_bld
-          CIBW_BEFORE_ALL_WINDOWS: nvcc -V &&
-                                   ci-utils\install_prereq_win.bat &&
-                                   xcopy /E /I /y local_install C:\TEMP\nyxus_bld\local_install
           CIBW_ENVIRONMENT_LINUX: LD_LIBRARY_PATH="/tmp/nyxus_bld/local_install/lib:/tmp/nyxus_bld/local_install/lib64:/usr/local/cuda/targets/x86_64-linux/lib:$LD_LIBRARY_PATH" CPATH="/usr/local/cuda/targets/x86_64-linux/include:$CPATH" PATH="/usr/local/cuda/bin:$PATH" NYXUS_DEP_DIR="/tmp/nyxus_bld/local_install" CXXFLAGS="-I /usr/local/cuda/include" CMAKE_ARGS="-DUSEGPU=ON -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc -DCMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES=/usr/local/cuda/include" NYXUS_GPU_WHEEL="ON" 
-          CIBW_ENVIRONMENT_WINDOWS: PATH="C:\\TEMP\\nyxus_bld\\local_install\\bin;$PATH" NYXUS_DEP_DIR="C:\\TEMP\\nyxus_bld\\local_install" CMAKE_ARGS="-DUSEGPU=ON" NYXUS_GPU_WHEEL="ON" 
           CIBW_REPAIR_WHEEL_COMMAND_LINUX: "auditwheel repair --exclude=libcufft.so --exclude=libcufft.so.10 --exclude=libcufft.so.10.4.2.109 --exclude=libcudart.so --exclude=libcudart.so.11.0 --exclude=libcudart.so.11.3.109  -w {dest_dir} {wheel}"
-          CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair -vv -w {dest_dir} {wheel} --no-dll=cufft64_10.dll;cufftw64_10.dll;cudart32_110.dll;cudart64_110.dll"
           CIBW_ARCHS: ${{ matrix.cibw_archs }}
           CIBW_TEST_REQUIRES: numpy pandas pyarrow pytest bfio
-          CIBW_BEFORE_TEST_WINDOWS: xcopy /E /I /y "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8\bin" %SystemRoot%\System32
+          CIBW_BEFORE_TEST_LINUX: >-
+            pip install "numpy<2.3.1" --only-binary=:all:
+          CIBW_TEST_REQUIRES_LINUX: pandas pyarrow pytest bfio
           CIBW_TEST_COMMAND: python -m pip check
 
+
       - name: Install Dependencies
         run: python -m pip install --upgrade twine requests
         
diff --git a/.github/workflows/publish_cuda12_pypi.yml b/.github/workflows/publish_cuda12_pypi.yml
@@ -12,7 +12,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [windows-2019, ubuntu-22.04]
+        os: [windows-2022, ubuntu-22.04]
         cibw_archs: ["auto64"]
         cibw_build: ["cp39-*", "cp310-*", "cp311-*", "cp312-*"]
 
@@ -28,8 +28,8 @@ jobs:
       - name: Install CUDA
         if: runner.os == 'Windows'
         env: 
-          cuda: "12.0.0"
-          visual_studio: "Visual Studio 16 2019"
+          cuda: "12.4.0"
+          visual_studio: "Visual Studio 17 2022"
         shell: powershell
         run: .\ci-utils\install_cuda_windows.ps1
 
@@ -58,10 +58,8 @@ jobs:
           CIBW_BUILD: ${{ matrix.cibw_build }}
           CIBW_SKIP: "*musllinux*"
           CIBW_BUILD_VERBOSITY: 3
-          CIBW_BEFORE_ALL_LINUX: sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo &&
-                                   sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo &&
-                                   sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo && 
-                                   yum install -y llvm libevent-devel openssl-devel && 
+          CIBW_MANYLINUX_X86_64_IMAGE: quay.io/pypa/manylinux_2_28_x86_64:latest
+          CIBW_BEFORE_ALL_LINUX: dnf -y install llvm libevent-devel openssl-devel && 
                                    bash ci-utils/install_cuda_yum.sh 12 && 
                                    bash ci-utils/install_prereq_linux.sh --build_arrow yes &&
                                    mkdir -p /tmp/nyxus_bld &&
@@ -75,7 +73,7 @@ jobs:
           CIBW_REPAIR_WHEEL_COMMAND_LINUX: auditwheel repair --exclude=libcufft.so --exclude=libcufft.so.11 --exclude=libcufft.so.11.0.12.1  --exclude=libcudart.so --exclude=libcudart.so.12 --exclude=libcudart.so.12.3.101  -w {dest_dir} {wheel}
           CIBW_ARCHS: ${{ matrix.cibw_archs }}
           CIBW_TEST_REQUIRES: numpy pandas pyarrow pytest bfio
-          CIBW_BEFORE_TEST_WINDOWS: xcopy /E /I /y "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.0\bin" %SystemRoot%\System32
+          CIBW_BEFORE_TEST_WINDOWS: xcopy /E /I /y "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\bin" %SystemRoot%\System32
           CIBW_TEST_COMMAND: python -m pip check
 
       - name: Install Dependencies
diff --git a/ci-utils/install_cuda_windows.ps1 b/ci-utils/install_cuda_windows.ps1
@@ -42,12 +42,13 @@ $CUDA_KNOWN_URLS = @{
     "11.7.0" = "https://developer.download.nvidia.com/compute/cuda/11.7.0/network_installers/cuda_11.7.0_windows_network.exe";
     "11.7.1" = "https://developer.download.nvidia.com/compute/cuda/11.7.1/network_installers/cuda_11.7.1_windows_network.exe";
     "11.8.0" = "https://developer.download.nvidia.com/compute/cuda/11.8.0/network_installers/cuda_11.8.0_windows_network.exe";
-    "12.0.0" = "https://developer.download.nvidia.com/compute/cuda/12.0.0/network_installers/cuda_12.0.0_windows_network.exe"
+    "12.0.0" = "https://developer.download.nvidia.com/compute/cuda/12.0.0/network_installers/cuda_12.0.0_windows_network.exe";
+    "12.4.0" = "https://developer.download.nvidia.com/compute/cuda/12.4.0/network_installers/cuda_12.4.0_windows_network.exe"
 }
 
 # @todo - change this to be based on _MSC_VER intead, or invert it to be CUDA keyed instead
 $VISUAL_STUDIO_MIN_CUDA = @{
-    "2022" = "11.6.0";
+    "2022" = "11.3.0";
     "2019" = "10.1";
     "2017" = "10.0"; # Depends on which version of 2017! 9.0 to 10.0 depending on version
     "2015" = "8.0";  # Might support older, unsure. Depracated as of 11.1, unsupported in 11.2
diff --git a/ci-utils/install_cuda_yum.sh b/ci-utils/install_cuda_yum.sh
@@ -1,23 +1,25 @@
 #!/bin/bash
 # Usage: $bash install_cuda_yum.sh <VERSION>
 
-yum-config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64/cuda-rhel7.repo &&
-yum clean all &&
-yum -y module install nvidia-driver:latest-dkms
-
 version=$1
 
 if [ $version -eq 11 ]; then
+    yum-config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64/cuda-rhel7.repo
+    yum clean all
+    yum -y module install nvidia-driver:latest-dkms
     echo "Installing cuda toolkit 11"
     yum -y install cuda-11-8
     echo 'export PATH=/usr/local/cuda-11.8/bin:$PATH' >> ~/.bashrc
     echo 'export LD_LIBRARY_PATH=/usr/local/cuda-11.8/lib64:$LD_LIBRARY_PATH' >> ~/.bashrc
     source ~/.bashrc
 elif [ $version -eq 12 ]; then
+    dnf config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel8/x86_64/cuda-rhel8.repo
+    dnf clean all
+    dnf -y module install nvidia-driver:latest-dkms
+    dnf -y install cuda-toolkit-12-8
     echo "Installing cuda toolkit 12"
-    yum -y install cuda-12-3
-    echo 'export PATH=/usr/local/cuda-12.3/bin:$PATH' >> ~/.bashrc
-    echo 'export LD_LIBRARY_PATH=/usr/local/cuda-12.3/lib64:$LD_LIBRARY_PATH' >> ~/.bashrc
+    echo 'export PATH=/usr/local/cuda-12.8/bin:$PATH' >> ~/.bashrc
+    echo 'export LD_LIBRARY_PATH=/usr/local/cuda-12.8/lib64:$LD_LIBRARY_PATH' >> ~/.bashrc
     source ~/.bashrc
 else
     echo "Invalid version. Please provide either 11 or 12."
diff --git a/ci-utils/install_prereq_win.bat b/ci-utils/install_prereq_win.bat
@@ -52,7 +52,7 @@ if "%BUILD_Z5_DEP%" == "1" (
     pushd boost_1_79_0 
     call bootstrap.bat 
     .\b2 headers --prefix=../local_install
-    .\b2 install --prefix=../local_install
+    .\b2 -j %NUMBER_OF_PROCESSORS% variant=release link=shared runtime-link=shared threading=multi install --prefix=../local_install
     xcopy /E /I /y boost ..\local_install\include\boost
     popd
 
