Skip to content

Commit 1fb62c6

Browse files
kosiewkosiew
committed
script to test various flags
1 parent f67eb94 commit 1fb62c6

File tree

1 file changed

+199
-0
lines changed

1 file changed

+199
-0
lines changed

‎test_benchmark_changes.sh‎

Lines changed: 199 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,199 @@
1+
#!/bin/bash
2+
# Test script for benchmark feature flags and CARGO_PROFILE changes
3+
# Commits: b7026e2c6^..f67eb945f
4+
5+
set -e
6+
7+
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
8+
cd "$SCRIPT_DIR/benchmarks"
9+
10+
# Binary is built at workspace root, not in benchmarks/
11+
DFBENCH="$SCRIPT_DIR/target/release/dfbench"
12+
13+
echo "========================================"
14+
echo "Testing Benchmark Feature Flag Changes"
15+
echo "========================================"
16+
echo ""
17+
18+
# Colors for output
19+
GREEN='\033[0;32m'
20+
RED='\033[0;31m'
21+
YELLOW='\033[1;33m'
22+
NC='\033[0m' # No Color
23+
24+
pass() {
25+
echo -e "${GREEN}✓${NC} $1"
26+
}
27+
28+
fail() {
29+
echo -e "${RED}✗${NC} $1"
30+
exit 1
31+
}
32+
33+
info() {
34+
echo -e "${YELLOW}➜${NC} $1"
35+
}
36+
37+
# Clean previous builds
38+
info "Cleaning previous builds..."
39+
cargo clean
40+
echo ""
41+
42+
# Test 1: Default build with all features
43+
echo "=== Test 1: Default build with all benchmarks enabled ==="
44+
info "Running: cargo build --release --bin dfbench"
45+
if cargo build --release --bin dfbench 2>&1 | tail -5; then
46+
pass "Default build succeeded"
47+
else
48+
fail "Default build failed"
49+
fi
50+
echo ""
51+
52+
# Test 2: Build with only bench-tpch
53+
echo "=== Test 2: Build with only bench-tpch feature ==="
54+
info "Running: cargo build --release --bin dfbench --no-default-features --features bench-tpch"
55+
if cargo build --release --bin dfbench --no-default-features --features "bench-tpch" 2>&1 | tail -5; then
56+
pass "Selective build (bench-tpch only) succeeded"
57+
else
58+
fail "Selective build failed"
59+
fi
60+
echo ""
61+
62+
# Test 3: Verify disabled benchmark shows proper error
63+
echo "=== Test 3: Verify disabled benchmark error message ==="
64+
info "First, rebuild with only bench-tpch to disable tpcds..."
65+
cargo build --release --bin dfbench --no-default-features --features "bench-tpch" > /dev/null 2>&1
66+
info "Now attempting to run disabled 'tpcds' benchmark (without --help)..."
67+
if "$DFBENCH" tpcds 2>&1 | grep -q "disabled.*bench-tpcds"; then
68+
pass "Disabled benchmark shows correct error message"
69+
else
70+
echo "Output was:"
71+
"$DFBENCH" tpcds 2>&1 | head -10
72+
fail "Disabled benchmark error message not working correctly"
73+
fi
74+
echo ""
75+
76+
# Test 4: Build with multiple specific features
77+
echo "=== Test 4: Build with multiple specific benchmarks ==="
78+
info "Running: cargo build --release --bin dfbench --no-default-features --features bench-tpch,bench-tpcds,mimalloc"
79+
if cargo build --release --bin dfbench --no-default-features --features "bench-tpch,bench-tpcds,mimalloc" 2>&1 | tail -5; then
80+
pass "Multi-feature build succeeded"
81+
else
82+
fail "Multi-feature build failed"
83+
fi
84+
echo ""
85+
86+
# Test 5: Verify enabled benchmarks work
87+
echo "=== Test 5: Verify enabled benchmarks show help ==="
88+
info "Using binary from Test 4 (built with bench-tpch,bench-tpcds,mimalloc)..."
89+
info "Testing tpch --help (should work)..."
90+
if "$DFBENCH" tpch --help > /dev/null 2>&1; then
91+
pass "Enabled benchmark (tpch) works"
92+
else
93+
fail "Enabled benchmark (tpch) doesn't work"
94+
fi
95+
96+
info "Testing tpcds --help (should work)..."
97+
if "$DFBENCH" tpcds --help > /dev/null 2>&1; then
98+
pass "Enabled benchmark (tpcds) works"
99+
else
100+
fail "Enabled benchmark (tpcds) doesn't work"
101+
fi
102+
103+
info "Testing clickbench (should fail with disabled message)..."
104+
if "$DFBENCH" clickbench 2>&1 | grep -q "disabled.*bench-clickbench"; then
105+
pass "Disabled benchmark (clickbench) shows error"
106+
else
107+
fail "Disabled benchmark (clickbench) should show error"
108+
fi
109+
echo ""
110+
111+
# Test 6: Test CARGO_PROFILE documentation
112+
echo "=== Test 6: Verify CARGO_PROFILE is documented in bench.sh ==="
113+
info "Checking bench.sh help output..."
114+
if ./bench.sh --help 2>&1 | grep -q "CARGO_PROFILE"; then
115+
pass "CARGO_PROFILE is documented in bench.sh"
116+
else
117+
fail "CARGO_PROFILE not found in bench.sh help"
118+
fi
119+
echo ""
120+
121+
# Test 7: Test dependency reduction
122+
echo "=== Test 7: Verify dependencies are reduced with selective features ==="
123+
info "Counting dependencies with bench-all..."
124+
FULL_DEPS=$(cargo tree --features "bench-all" -p datafusion-benchmarks 2>/dev/null | wc -l)
125+
info "Full dependencies: $FULL_DEPS lines"
126+
127+
info "Counting dependencies with only bench-tpch..."
128+
MINIMAL_DEPS=$(cargo tree --no-default-features --features "bench-tpch" -p datafusion-benchmarks 2>/dev/null | wc -l)
129+
info "Minimal dependencies: $MINIMAL_DEPS lines"
130+
131+
if [ "$MINIMAL_DEPS" -lt "$FULL_DEPS" ]; then
132+
pass "Selective features reduce dependency count ($MINIMAL_DEPS < $FULL_DEPS)"
133+
else
134+
fail "Selective features should reduce dependencies"
135+
fi
136+
echo ""
137+
138+
# Test 8: Run cargo test
139+
echo "=== Test 8: Run benchmark tests ==="
140+
info "Running: cargo test"
141+
if cargo test 2>&1 | tail -10; then
142+
pass "All tests passed"
143+
else
144+
fail "Some tests failed"
145+
fi
146+
echo ""
147+
148+
# Test 9: Verify all benchmark features individually
149+
echo "=== Test 9: Verify each benchmark feature compiles individually ==="
150+
FEATURES=(
151+
"bench-cancellation"
152+
"bench-clickbench"
153+
"bench-h2o"
154+
"bench-hj"
155+
"bench-imdb"
156+
"bench-nlj"
157+
"bench-smj"
158+
"bench-sort-tpch"
159+
"bench-tpch"
160+
"bench-tpcds"
161+
)
162+
163+
for feature in "${FEATURES[@]}"; do
164+
info "Testing feature: $feature"
165+
if cargo build --release --bin dfbench --no-default-features --features "$feature" 2>&1 | tail -1; then
166+
pass "Feature $feature builds successfully"
167+
else
168+
fail "Feature $feature failed to build"
169+
fi
170+
done
171+
echo ""
172+
173+
# Test 10: Verify README examples
174+
echo "=== Test 10: Verify README examples work ==="
175+
info "Testing: cargo run --profile release-nonlto --bin dfbench -- tpch --help"
176+
if cargo run --profile release-nonlto --bin dfbench -- tpch --help > /dev/null 2>&1; then
177+
pass "README example with release-nonlto profile works"
178+
else
179+
fail "README example with release-nonlto profile failed"
180+
fi
181+
echo ""
182+
183+
# Summary
184+
echo "========================================"
185+
echo "All Tests Passed Successfully! ✓"
186+
echo "========================================"
187+
echo ""
188+
echo "Summary of tested features:"
189+
echo " • Default build with bench-all"
190+
echo " • Selective feature builds"
191+
echo " • Disabled benchmark error messages"
192+
echo " • Multi-feature builds"
193+
echo " • CARGO_PROFILE documentation"
194+
echo " • Dependency reduction verification"
195+
echo " • Individual feature compilation"
196+
echo " • Test suite execution"
197+
echo " • README example validation"
198+
echo ""
199+
echo "All changes in commits b7026e2c6^..f67eb945f are working correctly!"

0 commit comments

Comments
 (0)