Skip to content

Commit 4f430e7

Browse files
ggerganovggerganov
authored
ci : add self-hosted workflows (#1357)
* ci : add self-hosted workflows * ci : sync env vars with llama.cpp * cont : always install python deps * cont : cancel ongoing runs * cont : exclude test-backend-ops from debug build
1 parent f3d690c commit 4f430e7

File tree

2 files changed

+277
-36
lines changed

2 files changed

+277
-36
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 212 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,10 @@ on:
66
pull_request:
77
branches: [ master ]
88

9+
concurrency:
10+
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
11+
cancel-in-progress: true
12+
913
jobs:
1014
build:
1115
strategy:
@@ -57,3 +61,211 @@ jobs:
5761
mkdir test-cmake
5862
cmake -S examples/test-cmake -B test-cmake -DCMAKE_PREFIX_PATH=${{ github.workspace }}/installed ${{ contains(matrix.os, 'windows') && '-A x64' || '-G Ninja -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++' }}
5963
cmake --build test-cmake ${{ contains(matrix.os, 'windows') && '--config Release' || '' }}
64+
65+
# TODO: simplify the following workflows using a matrix
66+
ggml-ci-x64-cpu-low-perf:
67+
runs-on: ubuntu-22.04
68+
69+
steps:
70+
- name: Clone
71+
id: checkout
72+
uses: actions/checkout@v4
73+
74+
- name: ccache
75+
uses: ggml-org/[email protected]
76+
with:
77+
key: ggml-ci-x64-cpu-low-perf
78+
evict-old-files: 1d
79+
80+
- name: Dependencies
81+
id: depends
82+
run: |
83+
sudo apt-get update
84+
sudo apt-get install build-essential libcurl4-openssl-dev
85+
86+
- name: Test
87+
id: ggml-ci
88+
run: |
89+
LLAMA_ARG_THREADS=$(nproc) GG_BUILD_LOW_PERF=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
90+
91+
ggml-ci-arm64-cpu-low-perf:
92+
runs-on: ubuntu-22.04-arm
93+
94+
steps:
95+
- name: Clone
96+
id: checkout
97+
uses: actions/checkout@v4
98+
99+
- name: ccache
100+
uses: ggml-org/[email protected]
101+
with:
102+
key: ggml-ci-arm64-cpu-low-perf
103+
evict-old-files: 1d
104+
105+
- name: Dependencies
106+
id: depends
107+
run: |
108+
sudo apt-get update
109+
sudo apt-get install build-essential libcurl4-openssl-dev
110+
111+
- name: Test
112+
id: ggml-ci
113+
run: |
114+
LLAMA_ARG_THREADS=$(nproc) GG_BUILD_LOW_PERF=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
115+
116+
ggml-ci-x64-cpu-high-perf:
117+
runs-on: ubuntu-22.04
118+
119+
steps:
120+
- name: Clone
121+
id: checkout
122+
uses: actions/checkout@v4
123+
124+
- name: ccache
125+
uses: ggml-org/[email protected]
126+
with:
127+
key: ggml-ci-x64-cpu-high-perf
128+
evict-old-files: 1d
129+
130+
- name: Dependencies
131+
id: depends
132+
run: |
133+
sudo apt-get update
134+
sudo apt-get install build-essential libcurl4-openssl-dev
135+
136+
- name: Test
137+
id: ggml-ci
138+
run: |
139+
LLAMA_ARG_THREADS=$(nproc) bash ./ci/run.sh ./tmp/results ./tmp/mnt
140+
141+
ggml-ci-arm64-cpu-high-perf:
142+
runs-on: ubuntu-22.04-arm
143+
144+
steps:
145+
- name: Clone
146+
id: checkout
147+
uses: actions/checkout@v4
148+
149+
- name: ccache
150+
uses: ggml-org/[email protected]
151+
with:
152+
key: ggml-ci-arm64-cpu-high-perf
153+
evict-old-files: 1d
154+
155+
- name: Dependencies
156+
id: depends
157+
run: |
158+
sudo apt-get update
159+
sudo apt-get install build-essential libcurl4-openssl-dev
160+
161+
- name: Test
162+
id: ggml-ci
163+
run: |
164+
LLAMA_ARG_THREADS=$(nproc) GG_BUILD_NO_SVE=1 GG_BUILD_NO_BF16=1 GG_BUILD_EXTRA_TESTS_0=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
165+
166+
ggml-ci-arm64-cpu-high-perf-sve:
167+
runs-on: ubuntu-22.04-arm
168+
169+
steps:
170+
- name: Clone
171+
id: checkout
172+
uses: actions/checkout@v4
173+
174+
- name: ccache
175+
uses: ggml-org/[email protected]
176+
with:
177+
key: ggml-ci-arm64-cpu-high-perf-sve
178+
evict-old-files: 1d
179+
180+
- name: Dependencies
181+
id: depends
182+
run: |
183+
sudo apt-get update
184+
sudo apt-get install build-essential libcurl4-openssl-dev
185+
186+
- name: Test
187+
id: ggml-ci
188+
run: |
189+
LLAMA_ARG_THREADS=$(nproc) GG_BUILD_NO_BF16=1 GG_BUILD_EXTRA_TESTS_0=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
190+
191+
ggml-ci-x64-nvidia-cuda:
192+
runs-on: [self-hosted, Linux, X64, NVIDIA]
193+
194+
steps:
195+
- name: Clone
196+
id: checkout
197+
uses: actions/checkout@v4
198+
199+
- name: Test
200+
id: ggml-ci
201+
run: |
202+
nvidia-smi
203+
GG_BUILD_CUDA=1 bash ./ci/run.sh ~/results/ggml /mnt/ggml
204+
205+
ggml-ci-x64-nvidia-vulkan-cm:
206+
runs-on: [self-hosted, Linux, X64, NVIDIA]
207+
208+
steps:
209+
- name: Clone
210+
id: checkout
211+
uses: actions/checkout@v4
212+
213+
- name: Test
214+
id: ggml-ci
215+
run: |
216+
vulkaninfo --summary
217+
GG_BUILD_VULKAN=1 GGML_VK_DISABLE_COOPMAT2=1 bash ./ci/run.sh ~/results/ggml /mnt/ggml
218+
219+
ggml-ci-x64-nvidia-vulkan-cm2:
220+
runs-on: [self-hosted, Linux, X64, NVIDIA, COOPMAT2]
221+
222+
steps:
223+
- name: Clone
224+
id: checkout
225+
uses: actions/checkout@v4
226+
227+
- name: Test
228+
id: ggml-ci
229+
run: |
230+
vulkaninfo --summary
231+
GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/ggml /mnt/ggml
232+
233+
ggml-ci-x64-cpu-amx:
234+
runs-on: [self-hosted, Linux, X64, CPU, AMX]
235+
236+
steps:
237+
- name: Clone
238+
id: checkout
239+
uses: actions/checkout@v4
240+
241+
- name: Test
242+
id: ggml-ci
243+
run: |
244+
bash ./ci/run.sh ~/results/ggml /mnt/ggml
245+
246+
ggml-ci-mac-metal:
247+
runs-on: [self-hosted, macOS, ARM64]
248+
249+
steps:
250+
- name: Clone
251+
id: checkout
252+
uses: actions/checkout@v4
253+
254+
- name: Test
255+
id: ggml-ci
256+
run: |
257+
GG_BUILD_METAL=1 bash ./ci/run.sh ~/results/ggml ~/mnt/ggml
258+
259+
ggml-ci-mac-vulkan:
260+
runs-on: [self-hosted, macOS, ARM64]
261+
262+
steps:
263+
- name: Clone
264+
id: checkout
265+
uses: actions/checkout@v4
266+
267+
- name: Test
268+
id: ggml-ci
269+
run: |
270+
vulkaninfo --summary
271+
GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/ggml ~/mnt/ggml

‎ci/run.sh‎

Lines changed: 65 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -36,15 +36,35 @@ SRC=`pwd`
3636
CMAKE_EXTRA=""
3737
CTEST_EXTRA=""
3838

39+
if [ ! -z ${GG_BUILD_METAL} ]; then
40+
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_METAL=ON"
41+
fi
42+
3943
if [ ! -z ${GG_BUILD_CUDA} ]; then
4044
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_CUDA=ON"
45+
46+
if command -v nvidia-smi >/dev/null 2>&1; then
47+
CUDA_ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader,nounits 2>/dev/null | head -1 | tr -d '.')
48+
if [[ -n "$CUDA_ARCH" && "$CUDA_ARCH" =~ ^[0-9]+$ ]]; then
49+
CMAKE_EXTRA="${CMAKE_EXTRA} -DCMAKE_CUDA_ARCHITECTURES=${CUDA_ARCH}"
50+
else
51+
echo "Warning: Using fallback CUDA architectures"
52+
CMAKE_EXTRA="${CMAKE_EXTRA} -DCMAKE_CUDA_ARCHITECTURES=61;70;75;80;86;89"
53+
fi
54+
else
55+
echo "Error: nvidia-smi not found, cannot build with CUDA"
56+
exit 1
57+
fi
4158
fi
4259

43-
if [ ! -z ${GG_BUILD_METAL} ]; then
44-
# TODO: this should use -DGGML_METAL_SHADER_DEBUG=ON instead, but currently it fails because
45-
# the binaries cannot locate default.metallib eventhough it is in bin/. cannot figure out
46-
# why this is happening, so temporary workaround is to use -DGGML_METAL_EMBED_LIBRARY=ON
47-
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_METAL=ON -DGGML_METAL_EMBED_LIBRARY=ON"
60+
if [ ! -z ${GG_BUILD_ROCM} ]; then
61+
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_HIP=ON"
62+
if [ -z ${GG_BUILD_AMDGPU_TARGETS} ]; then
63+
echo "Missing GG_BUILD_AMDGPU_TARGETS, please set it to your GPU architecture (e.g. gfx90a, gfx1100, etc.)"
64+
exit 1
65+
fi
66+
67+
CMAKE_EXTRA="${CMAKE_EXTRA} -DAMDGPU_TARGETS=${GG_BUILD_AMDGPU_TARGETS}"
4868
fi
4969

5070
if [ ! -z ${GG_BUILD_SYCL} ]; then
@@ -53,13 +73,40 @@ if [ ! -z ${GG_BUILD_SYCL} ]; then
5373
echo "source /opt/intel/oneapi/setvars.sh"
5474
exit 1
5575
fi
76+
# Use only main GPU
5677
export ONEAPI_DEVICE_SELECTOR="level_zero:0"
78+
# Enable sysman for correct memory reporting
5779
export ZES_ENABLE_SYSMAN=1
58-
# No plan to implement backward pass for now / disable test-opt
59-
CTEST_EXTRA="-E test-opt"
80+
# to circumvent precision issues on CPY operations
81+
export SYCL_PROGRAM_COMPILE_OPTIONS="-cl-fp32-correctly-rounded-divide-sqrt"
6082
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_SYCL=1 -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DGGML_SYCL_F16=ON"
6183
fi
6284

85+
if [ ! -z ${GG_BUILD_VULKAN} ]; then
86+
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_VULKAN=1"
87+
88+
# if on Mac, disable METAL
89+
if [[ "$OSTYPE" == "darwin"* ]]; then
90+
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_METAL=OFF -DGGML_BLAS=OFF"
91+
fi
92+
93+
fi
94+
95+
if [ ! -z ${GG_BUILD_WEBGPU} ]; then
96+
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_WEBGPU=1"
97+
fi
98+
99+
if [ ! -z ${GG_BUILD_MUSA} ]; then
100+
# Use qy1 by default (MTT S80)
101+
MUSA_ARCH=${MUSA_ARCH:-21}
102+
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_MUSA=ON -DMUSA_ARCHITECTURES=${MUSA_ARCH}"
103+
fi
104+
105+
if [ ! -z ${GG_BUILD_NO_SVE} ]; then
106+
# arm 9 and newer enables sve by default, adjust these flags depending on the cpu used
107+
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_NATIVE=OFF -DGGML_CPU_ARM_ARCH=armv8.5-a+fp16+i8mm"
108+
fi
109+
63110
## helpers
64111

65112
# download a file if it does not exist or if it is outdated
@@ -112,13 +159,9 @@ function gg_run_ctest_debug {
112159
set -e
113160

114161
(time cmake -DCMAKE_BUILD_TYPE=Debug ${CMAKE_EXTRA} .. ) 2>&1 | tee -a $OUT/${ci}-cmake.log
115-
(time make -j ) 2>&1 | tee -a $OUT/${ci}-make.log
116-
117-
if [ ! -z ${GG_BUILD_METAL} ]; then
118-
export GGML_METAL_PATH_RESOURCES="$(pwd)/bin"
119-
fi
162+
(time make -j$(nproc) ) 2>&1 | tee -a $OUT/${ci}-make.log
120163

121-
(time ctest ${CTEST_EXTRA} --output-on-failure -E test-opt ) 2>&1 | tee -a $OUT/${ci}-ctest.log
164+
(time ctest ${CTEST_EXTRA} --output-on-failure -E "test-opt|test-backend-ops" ) 2>&1 | tee -a $OUT/${ci}-ctest.log
122165

123166
set +e
124167
}
@@ -144,11 +187,7 @@ function gg_run_ctest_release {
144187
set -e
145188

146189
(time cmake -DCMAKE_BUILD_TYPE=Release ${CMAKE_EXTRA} .. ) 2>&1 | tee -a $OUT/${ci}-cmake.log
147-
(time make -j ) 2>&1 | tee -a $OUT/${ci}-make.log
148-
149-
if [ ! -z ${GG_BUILD_METAL} ]; then
150-
export GGML_METAL_PATH_RESOURCES="$(pwd)/bin"
151-
fi
190+
(time make -j$(nproc) ) 2>&1 | tee -a $OUT/${ci}-make.log
152191

153192
if [ -z $GG_BUILD_LOW_PERF ]; then
154193
(time ctest ${CTEST_EXTRA} --output-on-failure ) 2>&1 | tee -a $OUT/${ci}-ctest.log
@@ -316,7 +355,7 @@ function gg_sum_yolo {
316355

317356
## main
318357

319-
if [ -z ${GG_BUILD_LOW_PERF} ]; then
358+
if true ; then
320359
# Create symlink: ./ggml/models-mnt -> $MNT/models/models-mnt
321360
rm -rf ${SRC}/models-mnt
322361
mnt_models=${MNT}/models
@@ -335,27 +374,17 @@ fi
335374

336375

337376
ret=0
338-
if [ -z ${GG_BUILD_SYCL}]; then
339-
test $ret -eq 0 && gg_run ctest_debug
340-
fi
377+
test $ret -eq 0 && gg_run ctest_debug
341378
test $ret -eq 0 && gg_run ctest_release
342379

343-
if [ ! -z ${GG_BUILD_METAL} ]; then
344-
export GGML_METAL_PATH_RESOURCES="${SRC}/build-ci-release/bin"
345-
fi
346-
347-
if [ -z ${GG_BUILD_NO_DOWNLOAD} ]; then
348-
test $ret -eq 0 && gg_run gpt_2
349-
#test $ret -eq 0 && gg_run mnist
350-
test $ret -eq 0 && gg_run sam
351-
test $ret -eq 0 && gg_run yolo
352-
fi
380+
test $ret -eq 0 && gg_run gpt_2
381+
#test $ret -eq 0 && gg_run mnist
382+
test $ret -eq 0 && gg_run sam
383+
test $ret -eq 0 && gg_run yolo
353384

354385
if [ -z $GG_BUILD_LOW_PERF ]; then
355-
if [ -z ${GG_BUILD_VRAM_GB} ] || [ ${GG_BUILD_VRAM_GB} -ge 16 ]; then
356-
# run tests that require GPU with at least 16GB of VRAM
357-
date
358-
fi
386+
# run tests meant for low-perf runners
387+
date
359388
fi
360389

361390
exit $ret

0 commit comments

Comments
 (0)