Skip to content

Commit a5dd01b

Browse files
matthewdouglasmatthewdouglas
authored
Set up nightly CI for unit tests (#1619)
* Run unit tests on GH Actions * fix * fix * trigger workflow * Update * Update * Update * Run tests nightly * Disable paged optimizer test on Windows * Skip unit tests on Windows for CUDA 12.x (driver on runner is too old)
1 parent 10b9d4c commit a5dd01b

File tree

4 files changed

+202
-5
lines changed

4 files changed

+202
-5
lines changed

‎.github/scripts/build-cuda.sh‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,19 @@
22
declare build_arch
33
declare build_os
44
declare cuda_version
5+
declare cuda_targets
56

67
set -xeuo pipefail
78

8-
# By default, target Maxwell through Hopper.
9-
build_capability="50;52;60;61;70;75;80;86;89;90"
9+
if [[ -v cuda_targets ]]; then
10+
build_capability="${cuda_targets}"
11+
else
12+
# By default, target Maxwell through Hopper.
13+
build_capability="50;52;60;61;70;75;80;86;89;90"
1014

11-
# CUDA 12.8: Add sm100 and sm120; remove < sm75 to align with PyTorch 2.7+cu128 minimum
12-
[[ "${cuda_version}" == 12.8.* ]] && build_capability="75;80;86;89;90;100;120"
15+
# CUDA 12.8: Add sm100 and sm120; remove < sm75 to align with PyTorch 2.7+cu128 minimum
16+
[[ "${cuda_version}" == 12.8.* ]] && build_capability="75;80;86;89;90;100;120"
17+
fi
1318

1419
[[ "${build_os}" = windows-* ]] && python3 -m pip install ninja
1520

‎.github/workflows/tests.yml‎

Lines changed: 174 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,174 @@
1+
name: Unit tests
2+
3+
on:
4+
workflow_dispatch:
5+
schedule:
6+
# Every day at 02:15 AM UTC
7+
- cron: "15 2 * * *"
8+
push:
9+
branches: [testing-ci]
10+
11+
concurrency:
12+
group: ${{ github.workflow }}-${{ github.ref }}
13+
cancel-in-progress: true
14+
15+
jobs:
16+
17+
build-cpu:
18+
strategy:
19+
matrix:
20+
os: [ubuntu-22.04, windows-2025]
21+
arch: [x86_64]
22+
runs-on: ${{ matrix.os }}
23+
steps:
24+
- uses: actions/checkout@v4
25+
26+
- name: Setup MSVC
27+
if: startsWith(matrix.os, 'windows')
28+
uses: ilammy/[email protected] # to use cl
29+
30+
- name: Build C++
31+
run: bash .github/scripts/build-cpu.sh
32+
env:
33+
build_os: ${{ matrix.os }}
34+
build_arch: ${{ matrix.arch }}
35+
36+
- name: Upload build artifact
37+
uses: actions/upload-artifact@v4
38+
with:
39+
name: lib_cpu_${{ matrix.os }}_${{ matrix.arch }}
40+
path: output/${{ matrix.os }}/${{ matrix.arch }}/*
41+
retention-days: 7
42+
43+
build-cuda:
44+
strategy:
45+
matrix:
46+
cuda_version: ["11.8.0", "12.8.1"]
47+
os: [ubuntu-22.04, windows-2025]
48+
arch: [x86_64]
49+
runs-on: ${{ matrix.os }}
50+
51+
steps:
52+
- uses: actions/checkout@v4
53+
54+
- name: Install CUDA Toolkit
55+
uses: Jimver/[email protected]
56+
if: startsWith(matrix.os, 'windows')
57+
id: cuda-toolkit
58+
with:
59+
cuda: ${{ matrix.cuda_version }}
60+
method: "network"
61+
sub-packages: '["nvcc","cudart","cusparse","cublas","thrust","nvrtc_dev","cublas_dev","cusparse_dev"]'
62+
use-github-cache: false
63+
64+
- name: Setup MSVC
65+
if: startsWith(matrix.os, 'windows')
66+
uses: ilammy/[email protected] # to use cl
67+
68+
# We're running on T4 only for now, so we only target sm75.
69+
- name: Build C++ / CUDA
70+
run: bash .github/scripts/build-cuda.sh
71+
env:
72+
build_os: ${{ matrix.os }}
73+
build_arch: x86_64
74+
cuda_version: ${{ matrix.cuda_version }}
75+
cuda_targets: "75"
76+
77+
- name: Upload build artifact
78+
uses: actions/upload-artifact@v4
79+
with:
80+
name: lib_cuda_${{matrix.cuda_version}}_${{ matrix.os }}_${{ matrix.arch }}
81+
path: output/${{ matrix.os }}/${{ matrix.arch }}/*
82+
retention-days: 7
83+
84+
cpu-tests:
85+
needs: build-cpu
86+
strategy:
87+
fail-fast: false
88+
matrix:
89+
os: [ubuntu-22.04, windows-2025]
90+
arch: [x86_64]
91+
torch_version: ["2.7.0"]
92+
runs-on: ${{ matrix.os }}
93+
env:
94+
BNB_TEST_DEVICE: cpu
95+
steps:
96+
- uses: actions/checkout@v4
97+
98+
- name: Download build artifact
99+
uses: actions/download-artifact@v4
100+
with:
101+
name: lib_cpu_${{ matrix.os }}_${{ matrix.arch }}
102+
path: bitsandbytes/
103+
merge-multiple: true
104+
105+
- name: Setup Python
106+
uses: actions/setup-python@v5
107+
with:
108+
python-version: 3.9
109+
110+
- name: Install dependencies
111+
run: |
112+
pip install torch==${{ matrix.torch_version }} --index-url https://download.pytorch.org/whl/cpu
113+
pip install -e ".[test]"
114+
pip install pytest-cov
115+
116+
- name: Show installed packages
117+
run: pip list
118+
119+
- name: Run tests
120+
run: pytest
121+
122+
cuda-tests:
123+
needs: build-cuda
124+
strategy:
125+
fail-fast: false
126+
matrix:
127+
os: [ubuntu-22.04, windows-2025]
128+
arch: [x86_64]
129+
cuda_version: ["11.8.0", "12.8.1"]
130+
include:
131+
- cuda_version: "11.8.0"
132+
torch_version: "2.4.1"
133+
pypi_index: "https://download.pytorch.org/whl/cu118"
134+
- cuda_version: "12.8.1"
135+
torch_version: "2.7.0"
136+
pypi_index: "https://download.pytorch.org/whl/cu128"
137+
exclude:
138+
# Our current T4 Windows runner has a driver too old (471.11)
139+
# and cannot support CUDA 12+. Skip for now.
140+
- os: windows-2025
141+
cuda_version: "12.8.1"
142+
runs-on:
143+
labels: ${{ contains(matrix.os, 'windows') && 'CUDA-Windows-x64' || 'CUDA-Linux-x64' }}
144+
env:
145+
BNB_TEST_DEVICE: cuda
146+
steps:
147+
- name: Show GPU Information
148+
run: nvidia-smi
149+
150+
- uses: actions/checkout@v4
151+
152+
- name: Download build artifact
153+
uses: actions/download-artifact@v4
154+
with:
155+
name: lib_cuda_${{ matrix.cuda_version }}_${{ matrix.os }}_${{ matrix.arch }}
156+
path: bitsandbytes/
157+
merge-multiple: true
158+
159+
- name: Setup Python
160+
uses: actions/setup-python@v5
161+
with:
162+
python-version: 3.9
163+
164+
- name: Install dependencies
165+
run: |
166+
pip install torch==${{ matrix.torch_version }} --index-url ${{ matrix.pypi_index }}
167+
pip install -e ".[test]"
168+
pip install pytest-cov
169+
170+
- name: Show installed packages
171+
run: pip list
172+
173+
- name: Run tests
174+
run: pytest

‎tests/test_functional.py‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -728,6 +728,9 @@ def test_int8_double_quant(self, dim1, dim2):
728728
),
729729
)
730730
def test_integrated_int8_linear_matmul(self, device, dim1, dim4, inner):
731+
if device == "cpu" and inner > 2048:
732+
pytest.skip("Slow on CPU")
733+
731734
for i in range(k):
732735
A = torch.randn(dim1, inner, device=device).half()
733736
B = torch.randn(dim4, inner, device=device).half()
@@ -1316,7 +1319,18 @@ def test_gemv_4bit(self, device, dim, dtype, storage_type, quant_storage, double
13161319
if dtype == torch.float16:
13171320
if dim <= 512:
13181321
assert err1 < 7e-5
1319-
assert relerr1 < 0.0008
1322+
1323+
# TODO(matthewdouglas): On T4, dim=128-fp16-fc2-fp4-DQ will have relerror ~ 0.00092727
1324+
if (
1325+
device == "cuda"
1326+
and double_quant
1327+
and storage_type == "fp4"
1328+
and kind == "fc2"
1329+
and torch.cuda.get_device_capability() == (7, 5)
1330+
):
1331+
assert relerr1 < 0.00093
1332+
else:
1333+
assert relerr1 < 0.0008
13201334
else:
13211335
assert err1 < 6e-5
13221336
assert relerr1 < 2e-4

‎tests/test_optim.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import os
22
from os.path import join
33
import shutil
4+
import sys
45
import time
56
import uuid
67

@@ -168,6 +169,9 @@ def rm_path(path):
168169
@pytest.mark.parametrize("dim1", [1024], ids=id_formatter("dim1"))
169170
@pytest.mark.parametrize("dim2", [32, 1024, 4097, 1], ids=id_formatter("dim2"))
170171
def test_optimizer32bit(requires_cuda, dim1, dim2, gtype, optim_name):
172+
if optim_name.startswith("paged_") and sys.platform == "win32":
173+
pytest.skip("Paged optimizers can have issues on Windows.")
174+
171175
if gtype == torch.bfloat16 and optim_name in ["momentum", "rmsprop"]:
172176
pytest.skip()
173177
if dim1 == 1 and dim2 == 1:

0 commit comments

Comments
 (0)