Skip to content

Commit f11de0e

Browse files
redraskalredraskal
authored
ggml-ci: add run.sh (#2877)
1 parent d5cc27e commit f11de0e

File tree

2 files changed

+374
-0
lines changed

2 files changed

+374
-0
lines changed

‎ci/README.md‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
# CI
2+
3+
In addition to [Github Actions](https://github.com/ggerganov/whisper.cpp/actions) `whisper.cpp` uses a custom CI framework:
4+
5+
https://github.com/ggml-org/ci
6+
7+
It monitors the `master` branch for new commits and runs the
8+
[ci/run.sh](https://github.com/ggerganov/whisper.cpp/blob/master/ci/run.sh) script on dedicated cloud instances. This allows us
9+
to execute heavier workloads compared to just using Github Actions. Also with time, the cloud instances will be scaled
10+
to cover various hardware architectures, including GPU and Apple Silicon instances.
11+
12+
Collaborators can optionally trigger the CI run by adding the `ggml-ci` keyword to their commit message.
13+
Only the branches of this repo are monitored for this keyword.
14+
15+
It is a good practice, before publishing changes to execute the full CI locally on your machine:
16+
17+
```bash
18+
mkdir tmp
19+
20+
# CPU-only build
21+
bash ./ci/run.sh ./tmp/results ./tmp/mnt
22+
23+
# with CUDA support
24+
GGML_CUDA=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
25+
```
26+
27+
## Environment Variables
28+
29+
The CI script supports several environment variables to control the build:
30+
31+
| Variable | Description |
32+
|----------|-------------|
33+
| `GGML_CUDA` | Enable NVIDIA CUDA GPU acceleration |
34+
| `GGML_SYCL` | Enable Intel SYCL acceleration |
35+
| `GGML_VULKAN` | Enable Vulkan GPU acceleration |
36+
| `GGML_METAL` | Enable Metal acceleration on Apple Silicon |
37+
| `GGML_BLAS` | Enable BLAS CPU acceleration |
38+
| `WHISPER_OPENVINO` | Enable OpenVINO support |
39+
| `WHISPER_COREML` | Enable Core ML support for Apple Neural Engine |
40+
| `GG_BUILD_LOW_PERF` | Limit tests for low-performance hardware |
41+
| `GGML_TEST_MODELS` | Comma-separated list of models to test (e.g. "tiny.en,tiny,base,medium", defaults to all models unless `GG_BUILD_LOW_PERF` is set) |

‎ci/run.sh‎

Lines changed: 333 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,333 @@
1+
#!/bin/bash
2+
#
3+
# sample usage:
4+
#
5+
# mkdir tmp
6+
#
7+
# # CPU-only build
8+
# bash ./ci/run.sh ./tmp/results ./tmp/mnt
9+
#
10+
# # with CUDA support
11+
# GGML_CUDA=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
12+
#
13+
14+
if [ -z "$2" ]; then
15+
echo "usage: $0 <output-dir> <mnt-dir>"
16+
exit 1
17+
fi
18+
19+
mkdir -p "$1"
20+
mkdir -p "$2"
21+
22+
OUT=$(realpath "$1")
23+
MNT=$(realpath "$2")
24+
25+
rm -f "$OUT/*.log"
26+
rm -f "$OUT/*.exit"
27+
rm -f "$OUT/*.md"
28+
29+
sd=`dirname $0`
30+
cd $sd/../
31+
SRC=`pwd`
32+
33+
ALL_MODELS=( "tiny.en" "tiny" "base.en" "base" "small.en" "small" "medium.en" "medium" "large-v1" "large-v2" "large-v3" "large-v3-turbo" )
34+
BENCH_N_THREADS=4
35+
BENCH_ENCODER_ONLY=0
36+
BENCH_FLASH_ATTN=0
37+
38+
# check for user-specified models first. if not specified, use fast models
39+
if [ ! -z ${GGML_TEST_MODELS} ]; then
40+
IFS=',' read -r -a MODELS <<< "${GGML_TEST_MODELS}"
41+
else
42+
if [ ! -z ${GG_BUILD_LOW_PERF} ]; then
43+
MODELS=( "tiny" "base" "small" )
44+
else
45+
MODELS=("${ALL_MODELS[@]}")
46+
fi
47+
fi
48+
49+
CMAKE_EXTRA="-DWHISPER_FATAL_WARNINGS=ON"
50+
51+
if [ ! -z ${GGML_CUDA} ]; then
52+
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=native"
53+
fi
54+
55+
if [ ! -z ${GGML_SYCL} ]; then
56+
if [ -z ${ONEAPI_ROOT} ]; then
57+
echo "Not detected ONEAPI_ROOT, please install oneAPI base toolkit and enable it by:"
58+
echo "source /opt/intel/oneapi/setvars.sh"
59+
exit 1
60+
fi
61+
62+
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DGGML_SYCL_F16=ON"
63+
fi
64+
65+
if [ ! -z ${WHISPER_OPENVINO} ]; then
66+
CMAKE_EXTRA="${CMAKE_EXTRA} -DWHISPER_OPENVINO=ON"
67+
fi
68+
69+
if [ ! -z ${GGML_METAL} ]; then
70+
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_METAL=ON"
71+
fi
72+
73+
if [ ! -z ${GGML_VULKAN} ]; then
74+
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_VULKAN=ON"
75+
fi
76+
77+
if [ ! -z ${GGML_BLAS} ]; then
78+
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_BLAS=ON"
79+
fi
80+
81+
if [ ! -z ${WHISPER_COREML} ]; then
82+
CMAKE_EXTRA="${CMAKE_EXTRA} -DWHISPER_COREML=ON"
83+
fi
84+
85+
## helpers
86+
87+
# download a file if it does not exist or if it is outdated
88+
function gg_wget {
89+
local out=$1
90+
local url=$2
91+
92+
local cwd=`pwd`
93+
94+
mkdir -p $out
95+
cd $out
96+
97+
# should not re-download if file is the same
98+
wget -nv -N $url
99+
100+
cd $cwd
101+
}
102+
103+
function gg_download_model {
104+
local model_name=$1
105+
local model_file="$MNT/models/ggml-${model_name}.bin"
106+
107+
if [ ! -f ${model_file} ]; then
108+
local cwd=`pwd`
109+
mkdir -p "$MNT/models"
110+
cd "$MNT/models"
111+
bash "$cwd/models/download-ggml-model.sh" ${model_name} .
112+
cd "$cwd"
113+
fi
114+
}
115+
116+
function gg_printf {
117+
printf -- "$@" >> $OUT/README.md
118+
}
119+
120+
# Helper function to check command exit status
121+
function gg_check_last_command_status {
122+
local exit_file=$1
123+
local command_name=$2
124+
125+
local exit_status=$?
126+
echo "$exit_status" > "$exit_file"
127+
128+
if [ $exit_status -ne 0 ]; then
129+
echo "Error: Command $command_name failed with exit status $exit_status"
130+
return 1
131+
fi
132+
133+
return 0
134+
}
135+
136+
# Usage: gg_run <test_name> [additional_args...]
137+
#
138+
# Parameters:
139+
# test_name - Name of the test to run (calls gg_run_<test_name>)
140+
# additional_args - Any additional arguments to pass to the test function (first argument is appended to the log filename)
141+
function gg_run {
142+
ci=$1
143+
144+
if [ $# -gt 1 ]; then
145+
ci="${ci}_${2}"
146+
fi
147+
148+
set -o pipefail
149+
set -x
150+
151+
gg_run_$1 "$@" | tee $OUT/$ci.log
152+
cur=$?
153+
echo "$cur" > $OUT/$ci.exit
154+
155+
set +x
156+
set +o pipefail
157+
158+
gg_sum_$1 "$@"
159+
160+
ret=$((ret | cur))
161+
}
162+
163+
function gg_check_build_requirements {
164+
if ! command -v cmake &> /dev/null; then
165+
gg_printf 'cmake not found, please install'
166+
fi
167+
168+
if ! command -v make &> /dev/null; then
169+
gg_printf 'make not found, please install'
170+
fi
171+
}
172+
173+
## ci
174+
175+
function gg_run_ctest {
176+
mode=$2
177+
178+
cd ${SRC}
179+
180+
rm -rf build-ci-${mode} && mkdir build-ci-${mode} && cd build-ci-${mode}
181+
182+
set -e
183+
184+
gg_check_build_requirements
185+
186+
(time cmake -DCMAKE_BUILD_TYPE=${mode} ${CMAKE_EXTRA} .. ) 2>&1 | tee -a $OUT/${ci}-cmake.log
187+
(time make -j$(nproc) ) 2>&1 | tee -a $OUT/${ci}-make.log
188+
189+
(time ctest --output-on-failure -L main -E test-opt ) 2>&1 | tee -a $OUT/${ci}-ctest.log
190+
191+
set +e
192+
}
193+
194+
function gg_sum_ctest {
195+
mode=$2
196+
197+
gg_printf '### %s\n\n' "${ci}"
198+
199+
gg_printf 'Runs ctest in '${mode}' mode\n'
200+
gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
201+
gg_printf '```\n'
202+
gg_printf '%s\n' "$(cat $OUT/${ci}-ctest.log)"
203+
gg_printf '```\n'
204+
}
205+
206+
function gg_run_bench {
207+
cd ${SRC}
208+
209+
# set flash attention flag if enabled
210+
fattn=""
211+
if [ "$BENCH_FLASH_ATTN" -eq 1 ]; then
212+
fattn="-fa"
213+
fi
214+
215+
# run memcpy benchmark if not encoder-only mode
216+
if [ "$BENCH_ENCODER_ONLY" -eq 0 ]; then
217+
echo "Running memcpy benchmark"
218+
(time ./build-ci-release/bin/whisper-bench -w 1 -t $BENCH_N_THREADS 2>&1) | tee -a $OUT/${ci}-memcpy.log
219+
gg_check_last_command_status "$OUT/${ci}-memcpy.exit" "memcpy benchmark"
220+
221+
echo "Running ggml_mul_mat benchmark with $BENCH_N_THREADS threads"
222+
(time ./build-ci-release/bin/whisper-bench -w 2 -t $BENCH_N_THREADS 2>&1) | tee -a $OUT/${ci}-mul_mat.log
223+
gg_check_last_command_status "$OUT/${ci}-mul_mat.exit" "ggml_mul_mat benchmark"
224+
fi
225+
226+
echo "Running benchmark for all models"
227+
228+
# generate header for the benchmark table
229+
{
230+
printf "| %16s | %13s | %3s | %3s | %7s | %7s | %7s | %7s | %7s |\n" "Config" "Model" "Th" "FA" "Enc." "Dec." "Bch5" "PP" "Commit"
231+
printf "| %16s | %13s | %3s | %3s | %7s | %7s | %7s | %7s | %7s |\n" "---" "---" "---" "---" "---" "---" "---" "---" "---"
232+
} | tee -a $OUT/${ci}-models-table.log
233+
234+
# run benchmark for each model
235+
for model in "${MODELS[@]}"; do
236+
echo "Benchmarking model: $model"
237+
238+
# run the benchmark and capture output
239+
output=$(./build-ci-release/bin/whisper-bench -m $MNT/models/ggml-$model.bin -t $BENCH_N_THREADS $fattn 2>&1)
240+
ret=$?
241+
242+
# save the raw output
243+
echo "$output" > $OUT/${ci}-bench-$model.log
244+
245+
if [ $ret -eq 0 ]; then
246+
# parse the benchmark results
247+
encode_time=$(echo "$output" | grep "encode time" | awk '{print $11}')
248+
decode_time=$(echo "$output" | grep "decode time" | awk '{print $11}')
249+
batchd_time=$(echo "$output" | grep "batchd time" | awk '{print $11}')
250+
prompt_time=$(echo "$output" | grep "prompt time" | awk '{print $11}')
251+
system_info=$(echo "$output" | grep "system_info")
252+
actual_threads=$(echo "$output" | grep "system_info" | awk '{print $4}')
253+
254+
# determine configuration
255+
config=""
256+
if [[ $system_info == *"AVX2 = 1"* ]]; then
257+
config="$config AVX2"
258+
fi
259+
if [[ $system_info == *"NEON = 1"* ]]; then
260+
config="$config NEON"
261+
fi
262+
if [[ $system_info == *"BLAS = 1"* ]]; then
263+
config="$config BLAS"
264+
fi
265+
if [[ $system_info == *"COREML = 1"* ]]; then
266+
config="$config COREML"
267+
fi
268+
if [[ $system_info == *"CUDA = 1"* ]]; then
269+
config="$config CUDA"
270+
fi
271+
if [[ $system_info == *"METAL = 1"* ]]; then
272+
config="$config METAL"
273+
fi
274+
275+
# get commit hash
276+
commit=$(git rev-parse --short HEAD)
277+
278+
# add row to benchmark table
279+
printf "| %16s | %13s | %3s | %3s | %7s | %7s | %7s | %7s | %7s |\n" \
280+
"$config" "$model" "$actual_threads" "$BENCH_FLASH_ATTN" "$encode_time" "$decode_time" "$batchd_time" "$prompt_time" "$commit" \
281+
| tee -a $OUT/${ci}-models-table.log
282+
else
283+
echo "Benchmark failed for model: $model" | tee -a $OUT/${ci}-bench-errors.log
284+
fi
285+
done
286+
}
287+
288+
function gg_sum_bench {
289+
gg_printf '### %s\n\n' "${ci}"
290+
291+
gg_printf 'Whisper Benchmark Results\n'
292+
gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
293+
294+
# show memcpy and ggml_mul_mat benchmark results if available
295+
if [ "$BENCH_ENCODER_ONLY" -eq 0 ]; then
296+
if [ -f "$OUT/${ci}-memcpy.log" ]; then
297+
gg_printf '#### memcpy Benchmark\n\n'
298+
gg_printf '```\n%s\n```\n\n' "$(cat $OUT/${ci}-memcpy.log)"
299+
fi
300+
301+
if [ -f "$OUT/${ci}-mul_mat.log" ]; then
302+
gg_printf '#### ggml_mul_mat Benchmark\n\n'
303+
gg_printf '```\n%s\n```\n\n' "$(cat $OUT/${ci}-mul_mat.log)"
304+
fi
305+
fi
306+
307+
# show model benchmark results
308+
gg_printf '#### Model Benchmarks\n\n'
309+
if [ -f "$OUT/${ci}-models-table.log" ]; then
310+
gg_printf '%s\n\n' "$(cat $OUT/${ci}-models-table.log)"
311+
else
312+
gg_printf 'No model benchmark results available.\n\n'
313+
fi
314+
315+
# show any errors that occurred
316+
if [ -f "$OUT/${ci}-bench-errors.log" ]; then
317+
gg_printf '#### Benchmark Errors\n\n'
318+
gg_printf '```\n%s\n```\n\n' "$(cat $OUT/${ci}-bench-errors.log)"
319+
fi
320+
}
321+
322+
ret=0
323+
324+
for model in "${MODELS[@]}"; do
325+
test $ret -eq 0 && gg_download_model ${model}
326+
done
327+
328+
test $ret -eq 0 && gg_run ctest debug
329+
test $ret -eq 0 && gg_run ctest release
330+
331+
test $ret -eq 0 && gg_run bench
332+
333+
exit $ret

0 commit comments

Comments
 (0)