|
| 1 | +#!/usr/bin/env bash |
| 2 | +set -euo pipefail |
| 3 | + |
| 4 | +usage() { |
| 5 | + cat <<'USAGE' |
| 6 | +Usage: run-manual-regression.sh [options] |
| 7 | +
|
| 8 | +Options: |
| 9 | + --reference-ref <ref> Git ref (commit/tag) for reference build (default: HEAD~1) |
| 10 | + --snr-min <value> Minimum SNR threshold (default: 60) |
| 11 | + --rms-max <value> Maximum RMS difference (default: 0.0001) |
| 12 | + --abs-max <value> Maximum absolute difference (default: 0.01) |
| 13 | + --build-type <type> CMake build type (default: RelWithDebInfo) |
| 14 | + --cmake-flags <flags> Extra flags passed to both CMake configure steps |
| 15 | + --allow-missing Allow missing files while still comparing the rest |
| 16 | + --keep-worktree Keep the reference worktree after running |
| 17 | + --help Show this help message |
| 18 | +
|
| 19 | +Environment variables: |
| 20 | + REFERENCE_REF, SNR_MIN, RMS_MAX, ABS_MAX, BUILD_TYPE, REGRESSION_CMAKE_FLAGS, ALLOW_MISSING |
| 21 | +USAGE |
| 22 | +} |
| 23 | + |
| 24 | +REFERENCE_REF=${REFERENCE_REF:-HEAD~1} |
| 25 | +SNR_MIN=${SNR_MIN:-60} |
| 26 | +RMS_MAX=${RMS_MAX:-0.0001} |
| 27 | +ABS_MAX=${ABS_MAX:-0.01} |
| 28 | +BUILD_TYPE=${BUILD_TYPE:-RelWithDebInfo} |
| 29 | +REGRESSION_CMAKE_FLAGS=${REGRESSION_CMAKE_FLAGS:-} |
| 30 | +ALLOW_MISSING=${ALLOW_MISSING:-0} |
| 31 | +KEEP_WORKTREE=0 |
| 32 | + |
| 33 | +while [[ $# -gt 0 ]]; do |
| 34 | + case "$1" in |
| 35 | + --reference-ref) |
| 36 | + REFERENCE_REF="$2" |
| 37 | + shift 2 |
| 38 | + ;; |
| 39 | + --snr-min) |
| 40 | + SNR_MIN="$2" |
| 41 | + shift 2 |
| 42 | + ;; |
| 43 | + --rms-max) |
| 44 | + RMS_MAX="$2" |
| 45 | + shift 2 |
| 46 | + ;; |
| 47 | + --abs-max) |
| 48 | + ABS_MAX="$2" |
| 49 | + shift 2 |
| 50 | + ;; |
| 51 | + --build-type) |
| 52 | + BUILD_TYPE="$2" |
| 53 | + shift 2 |
| 54 | + ;; |
| 55 | + --cmake-flags) |
| 56 | + REGRESSION_CMAKE_FLAGS="$2" |
| 57 | + shift 2 |
| 58 | + ;; |
| 59 | + --allow-missing) |
| 60 | + ALLOW_MISSING=1 |
| 61 | + shift 1 |
| 62 | + ;; |
| 63 | + --keep-worktree) |
| 64 | + KEEP_WORKTREE=1 |
| 65 | + shift 1 |
| 66 | + ;; |
| 67 | + --help) |
| 68 | + usage |
| 69 | + exit 0 |
| 70 | + ;; |
| 71 | + *) |
| 72 | + echo "Unknown option: $1" >&2 |
| 73 | + usage |
| 74 | + exit 1 |
| 75 | + ;; |
| 76 | + esac |
| 77 | +done |
| 78 | + |
| 79 | +if ! command -v sox >/dev/null 2>&1; then |
| 80 | + echo "sox is required to compare audio files." >&2 |
| 81 | + exit 1 |
| 82 | +fi |
| 83 | +if ! command -v cmake >/dev/null 2>&1; then |
| 84 | + echo "cmake is required to build fluidsynth." >&2 |
| 85 | + exit 1 |
| 86 | +fi |
| 87 | +if ! command -v git >/dev/null 2>&1; then |
| 88 | + echo "git is required to check out the reference revision." >&2 |
| 89 | + exit 1 |
| 90 | +fi |
| 91 | + |
| 92 | +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) |
| 93 | +ROOT_DIR=$(cd "${SCRIPT_DIR}/.." && pwd) |
| 94 | + |
| 95 | +if ! git -C "$ROOT_DIR" rev-parse --is-inside-work-tree >/dev/null 2>&1; then |
| 96 | + echo "${ROOT_DIR} is not a git repository." >&2 |
| 97 | + exit 1 |
| 98 | +fi |
| 99 | + |
| 100 | +if ! git -C "$ROOT_DIR" rev-parse --verify "$REFERENCE_REF" >/dev/null 2>&1; then |
| 101 | + echo "Reference ref ${REFERENCE_REF} not found. Fetch full history or set REFERENCE_REF." >&2 |
| 102 | + exit 1 |
| 103 | +fi |
| 104 | + |
| 105 | +CMAKE_GENERATOR_ARGS=() |
| 106 | +if [[ -n "${CMAKE_GENERATOR:-}" ]]; then |
| 107 | + CMAKE_GENERATOR_ARGS=(-G "$CMAKE_GENERATOR") |
| 108 | +elif command -v ninja >/dev/null 2>&1; then |
| 109 | + CMAKE_GENERATOR_ARGS=(-G Ninja) |
| 110 | +fi |
| 111 | + |
| 112 | +CMAKE_FLAGS=() |
| 113 | +if [[ -n "$REGRESSION_CMAKE_FLAGS" ]]; then |
| 114 | + read -r -a CMAKE_FLAGS <<< "$REGRESSION_CMAKE_FLAGS" |
| 115 | +fi |
| 116 | + |
| 117 | +CURRENT_BUILD_DIR="${CURRENT_BUILD_DIR:-${ROOT_DIR}/build/regression-current}" |
| 118 | +REFERENCE_TEST_BUILD_DIR="${REFERENCE_TEST_BUILD_DIR:-${ROOT_DIR}/build/regression-reference}" |
| 119 | + |
| 120 | +REF_WORKTREE=$(mktemp -d "${TMPDIR:-/tmp}/fluidsynth-reference-XXXXXX") |
| 121 | +cleanup() { |
| 122 | + if [[ $KEEP_WORKTREE -eq 0 ]]; then |
| 123 | + git -C "$ROOT_DIR" worktree remove --force "$REF_WORKTREE" >/dev/null 2>&1 || true |
| 124 | + rm -rf "$REF_WORKTREE" |
| 125 | + git -C "$ROOT_DIR" worktree prune >/dev/null 2>&1 || true |
| 126 | + fi |
| 127 | +} |
| 128 | +trap cleanup EXIT |
| 129 | + |
| 130 | +echo "Checking out reference revision ${REFERENCE_REF}..." |
| 131 | +git -C "$ROOT_DIR" worktree add --detach "$REF_WORKTREE" "$REFERENCE_REF" >/dev/null |
| 132 | + |
| 133 | +REF_BUILD_DIR="${REF_WORKTREE}/build-regression" |
| 134 | +cmake -S "$REF_WORKTREE" -B "$REF_BUILD_DIR" "${CMAKE_GENERATOR_ARGS[@]}" \ |
| 135 | + -DCMAKE_BUILD_TYPE="$BUILD_TYPE" \ |
| 136 | + "${CMAKE_FLAGS[@]}" |
| 137 | +cmake --build "$REF_BUILD_DIR" --target fluidsynth |
| 138 | + |
| 139 | +REF_FLUIDSYNTH=$(find "$REF_BUILD_DIR" -type f -name fluidsynth -perm -111 | head -n 1) |
| 140 | +if [[ -z "$REF_FLUIDSYNTH" ]]; then |
| 141 | + echo "Could not locate reference fluidsynth binary." >&2 |
| 142 | + exit 1 |
| 143 | +fi |
| 144 | + |
| 145 | +cmake -S "$ROOT_DIR" -B "$CURRENT_BUILD_DIR" "${CMAKE_GENERATOR_ARGS[@]}" \ |
| 146 | + -DCMAKE_BUILD_TYPE="$BUILD_TYPE" \ |
| 147 | + "${CMAKE_FLAGS[@]}" |
| 148 | +cmake --build "$CURRENT_BUILD_DIR" --target check_rendering --parallel $(nproc) |
| 149 | + |
| 150 | +cmake -S "$ROOT_DIR" -B "$REFERENCE_TEST_BUILD_DIR" "${CMAKE_GENERATOR_ARGS[@]}" \ |
| 151 | + -DCMAKE_BUILD_TYPE="$BUILD_TYPE" \ |
| 152 | + -DMANUAL_TEST_FLUIDSYNTH="$REF_FLUIDSYNTH" \ |
| 153 | + "${CMAKE_FLAGS[@]}" |
| 154 | +cmake --build "$REFERENCE_TEST_BUILD_DIR" --target check_rendering --parallel $(nproc) || true |
| 155 | + |
| 156 | +CURRENT_OUTPUT_DIR="${CURRENT_BUILD_DIR}/test/manual" |
| 157 | +REFERENCE_OUTPUT_DIR="${REFERENCE_TEST_BUILD_DIR}/test/manual" |
| 158 | + |
| 159 | +if [[ ! -d "$CURRENT_OUTPUT_DIR" || ! -d "$REFERENCE_OUTPUT_DIR" ]]; then |
| 160 | + echo "Manual test output directories were not created." >&2 |
| 161 | + exit 1 |
| 162 | +fi |
| 163 | + |
| 164 | +mapfile -d '' current_files < <(find "$CURRENT_OUTPUT_DIR" -type f \( -name "*.wav" -o -name "*.raw" \) -print0 | LC_ALL=C sort -z) |
| 165 | +mapfile -d '' reference_files < <(find "$REFERENCE_OUTPUT_DIR" -type f \( -name "*.wav" -o -name "*.raw" \) -print0 | LC_ALL=C sort -z) |
| 166 | + |
| 167 | +if [[ ${#current_files[@]} -eq 0 ]]; then |
| 168 | + echo "No rendered audio files found in ${CURRENT_OUTPUT_DIR}." >&2 |
| 169 | + exit 1 |
| 170 | +fi |
| 171 | + |
| 172 | +missing_count=0 |
| 173 | +for ref_file in "${reference_files[@]}"; do |
| 174 | + rel_path=${ref_file#"$REFERENCE_OUTPUT_DIR/"} |
| 175 | + if [[ ! -f "$CURRENT_OUTPUT_DIR/$rel_path" ]]; then |
| 176 | + echo "Missing current render for ${rel_path}" >&2 |
| 177 | + missing_count=$((missing_count + 1)) |
| 178 | + fi |
| 179 | +done |
| 180 | + |
| 181 | +for current_file in "${current_files[@]}"; do |
| 182 | + rel_path=${current_file#"$CURRENT_OUTPUT_DIR/"} |
| 183 | + if [[ ! -f "$REFERENCE_OUTPUT_DIR/$rel_path" ]]; then |
| 184 | + echo "Missing reference render for ${rel_path}" >&2 |
| 185 | + missing_count=$((missing_count + 1)) |
| 186 | + fi |
| 187 | +done |
| 188 | + |
| 189 | +extract_stat() { |
| 190 | + local pattern="$1" |
| 191 | + awk -F: -v pat="$pattern" '$1 ~ pat { gsub(/^[ \t]+/, "", $2); print $2; exit }' |
| 192 | +} |
| 193 | + |
| 194 | +failures=0 |
| 195 | +compared=0 |
| 196 | +missing_fail=0 |
| 197 | +printf "%-70s %12s %12s %12s\n" "File" "SNR" "RMS" "ABS" |
| 198 | +for current_file in "${current_files[@]}"; do |
| 199 | + rel_path=${current_file#"$CURRENT_OUTPUT_DIR/"} |
| 200 | + reference_file="$REFERENCE_OUTPUT_DIR/$rel_path" |
| 201 | + if [[ ! -f "$current_file" ]]; then |
| 202 | + continue |
| 203 | + fi |
| 204 | + if [[ ! -f "$reference_file" ]]; then |
| 205 | + continue |
| 206 | + fi |
| 207 | + compared=$((compared + 1)) |
| 208 | + |
| 209 | + signal_stats=$(sox "$reference_file" -n stat 2>&1) |
| 210 | + rms_signal=$(printf '%s\n' "$signal_stats" | extract_stat "RMS[[:space:]]+amplitude") |
| 211 | + |
| 212 | + diff_stats=$(sox -m -v 1 "$current_file" -v -1 "$reference_file" -n stat 2>&1) |
| 213 | + rms_diff=$(printf '%s\n' "$diff_stats" | extract_stat "RMS[[:space:]]+amplitude") |
| 214 | + abs_diff=$(printf '%s\n' "$diff_stats" | extract_stat "Maximum amplitude") |
| 215 | + |
| 216 | + snr_value=$(awk -v signal="$rms_signal" -v noise="$rms_diff" 'BEGIN { if (noise == 0) { print 1e9; } else { print 20*log(signal/noise)/log(10); } }') |
| 217 | + snr_display=$(awk -v value="$snr_value" 'BEGIN { if (value > 1e8) { print "inf"; } else { printf "%.2f", value; } }') |
| 218 | + |
| 219 | + printf "%-70s %12s %12s %12s\n" "$rel_path" "$snr_display" "$rms_diff" "$abs_diff" |
| 220 | + |
| 221 | + if ! awk -v value="$snr_value" -v min="$SNR_MIN" 'BEGIN { exit !(value >= min) }'; then |
| 222 | + echo " SNR below threshold (${SNR_MIN})" >&2 |
| 223 | + failures=$((failures + 1)) |
| 224 | + fi |
| 225 | + if ! awk -v value="$rms_diff" -v max="$RMS_MAX" 'BEGIN { exit !(value <= max) }'; then |
| 226 | + echo " RMS above threshold (${RMS_MAX})" >&2 |
| 227 | + failures=$((failures + 1)) |
| 228 | + fi |
| 229 | + if ! awk -v value="$abs_diff" -v max="$ABS_MAX" 'BEGIN { exit !(value <= max) }'; then |
| 230 | + echo " ABS above threshold (${ABS_MAX})" >&2 |
| 231 | + failures=$((failures + 1)) |
| 232 | + fi |
| 233 | +done |
| 234 | + |
| 235 | +if [[ $missing_count -ne 0 ]]; then |
| 236 | + if [[ $ALLOW_MISSING -eq 0 ]]; then |
| 237 | + echo "Missing file pairs: ${missing_count}" >&2 |
| 238 | + missing_fail=1 |
| 239 | + else |
| 240 | + echo "Missing file pairs: ${missing_count} (allowed)" >&2 |
| 241 | + fi |
| 242 | +fi |
| 243 | + |
| 244 | +if [[ $compared -eq 0 ]]; then |
| 245 | + if [[ $missing_count -ne 0 ]]; then |
| 246 | + if [[ $ALLOW_MISSING -eq 0 ]]; then |
| 247 | + echo "No matching audio files to compare because file pairs are missing." >&2 |
| 248 | + else |
| 249 | + echo "No matching audio files to compare; only missing pairs were found." >&2 |
| 250 | + fi |
| 251 | + else |
| 252 | + echo "No matching audio files to compare." >&2 |
| 253 | + fi |
| 254 | + exit 1 |
| 255 | +fi |
| 256 | + |
| 257 | +if [[ $failures -ne 0 || $missing_fail -ne 0 ]]; then |
| 258 | + if [[ $missing_fail -ne 0 && $failures -ne 0 ]]; then |
| 259 | + echo "Audio regression check failed with ${failures} threshold violations and ${missing_count} missing pairs." >&2 |
| 260 | + elif [[ $missing_fail -ne 0 ]]; then |
| 261 | + echo "Audio regression check failed with ${missing_count} missing pairs." >&2 |
| 262 | + else |
| 263 | + echo "Audio regression check failed with ${failures} threshold violations." >&2 |
| 264 | + fi |
| 265 | + exit 1 |
| 266 | +fi |
| 267 | + |
| 268 | +echo "Audio regression check passed." |
0 commit comments