Skip to content

Commit ef4fd90

Browse files
Thomas StrombergThomas Stromberg
Thomas Stromberg
authored and
Thomas Stromberg
committed
Release v1.9.0
1 parent eb6bb4a commit ef4fd90

File tree

9 files changed

+349
-7
lines changed

9 files changed

+349
-7
lines changed

‎benchmarks/tune_ghost_cap.sh‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
#!/bin/bash
2+
# Binary search for optimal ghostCapPerMille across cache sizes
3+
set -e
4+
5+
S3FIFO="$(dirname "$0")/../s3fifo.go"
6+
SIZE="${1:-16}"
7+
RESULTS_FILE="/tmp/ghost_cap_${SIZE}k.txt"
8+
9+
# Clear previous results
10+
> "$RESULTS_FILE"
11+
12+
run_benchmark() {
13+
local cap=$1
14+
15+
# Skip if already tested
16+
if grep -q "^$cap " "$RESULTS_FILE" 2>/dev/null; then
17+
return 0
18+
fi
19+
20+
# Update ghostCapPerMille in s3fifo.go
21+
sed -i '' "s/ghostCapPerMille = [0-9]*/ghostCapPerMille = $cap/" "$S3FIFO"
22+
23+
# Run benchmark - capture output
24+
output=$(SIZES=$SIZE SUITES=hitrate make benchmark 2>&1) || true
25+
26+
# Extract the average from Category Summary line for multicache
27+
avg=$(echo "$output" | grep -E '^\s*\| multicache' | tail -1 | awk -F'|' '{print $3}' | grep -o '[0-9.]*')
28+
29+
if [ -z "$avg" ]; then
30+
echo "cap=$cap: FAILED"
31+
return 1
32+
fi
33+
34+
echo "cap=$cap ($(echo "scale=2; $cap/10" | bc)%): hitrate=$avg%"
35+
echo "$cap $avg" >> "$RESULTS_FILE"
36+
}
37+
38+
echo "=== Tuning ghostCapPerMille for ${SIZE}K cache ==="
39+
echo ""
40+
41+
# Initial coarse scan (500 = 0.5x to 2000 = 2.0x)
42+
echo "=== Coarse Scan ==="
43+
for cap in 500 750 1000 1220 1500 1750 2000; do
44+
run_benchmark $cap
45+
done
46+
47+
echo ""
48+
echo "=== Results So Far ==="
49+
sort -k2 -rn "$RESULTS_FILE" | head -10
50+
51+
# Find best from initial scan
52+
best_line=$(sort -k2 -rn "$RESULTS_FILE" | head -1)
53+
best_cap=$(echo "$best_line" | awk '{print $1}')
54+
55+
echo ""
56+
echo "Best so far: cap=$best_cap"
57+
58+
# Fine-tune around best value
59+
echo ""
60+
echo "=== Fine-tuning around $best_cap ==="
61+
low=$((best_cap - 300))
62+
high=$((best_cap + 300))
63+
[ $low -lt 100 ] && low=100
64+
65+
for cap in $(seq $low 50 $high); do
66+
run_benchmark $cap
67+
done
68+
69+
echo ""
70+
echo "=== Final Results (sorted by hitrate) ==="
71+
sort -k2 -rn "$RESULTS_FILE" | head -15
72+
73+
# Find the actual best
74+
best_line=$(sort -k2 -rn "$RESULTS_FILE" | head -1)
75+
best_cap=$(echo "$best_line" | awk '{print $1}')
76+
best_hitrate=$(echo "$best_line" | awk '{print $2}')
77+
78+
echo ""
79+
echo "=== OPTIMAL for ${SIZE}K ==="
80+
echo "ghostCapPerMille = $best_cap ($(echo "scale=2; $best_cap/10" | bc)%) with hitrate=$best_hitrate%"
81+
82+
# Restore original value
83+
sed -i '' "s/ghostCapPerMille = [0-9]*/ghostCapPerMille = 1220/" "$S3FIFO"
84+
echo ""
85+
echo "Restored ghostCapPerMille to 1220"

‎benchmarks/tune_max_freq.sh‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
#!/bin/bash
2+
# Binary search for optimal maxFreq across cache sizes
3+
set -e
4+
5+
S3FIFO="$(dirname "$0")/../s3fifo.go"
6+
SIZE="${1:-16}"
7+
RESULTS_FILE="/tmp/max_freq_${SIZE}k.txt"
8+
9+
# Clear previous results
10+
> "$RESULTS_FILE"
11+
12+
run_benchmark() {
13+
local freq=$1
14+
15+
# Skip if already tested
16+
if grep -q "^$freq " "$RESULTS_FILE" 2>/dev/null; then
17+
return 0
18+
fi
19+
20+
# Update maxFreq in s3fifo.go
21+
sed -i '' "s/maxFreq = [0-9]*/maxFreq = $freq/" "$S3FIFO"
22+
23+
# Run benchmark - capture output
24+
output=$(SIZES=$SIZE SUITES=hitrate make benchmark 2>&1) || true
25+
26+
# Extract the average from Category Summary line for multicache
27+
avg=$(echo "$output" | grep -E '^\s*\| multicache' | tail -1 | awk -F'|' '{print $3}' | grep -o '[0-9.]*')
28+
29+
if [ -z "$avg" ]; then
30+
echo "freq=$freq: FAILED"
31+
return 1
32+
fi
33+
34+
echo "maxFreq=$freq: hitrate=$avg%"
35+
echo "$freq $avg" >> "$RESULTS_FILE"
36+
}
37+
38+
echo "=== Tuning maxFreq for ${SIZE}K cache ==="
39+
echo ""
40+
41+
# Test range from 2 (minimum safe) to 10
42+
# Note: maxFreq=1 causes infinite loop, so minimum is 2
43+
echo "=== Testing maxFreq 2-10 ==="
44+
for freq in 2 3 4 5 6 7 8 9 10; do
45+
run_benchmark $freq
46+
done
47+
48+
echo ""
49+
echo "=== Results (sorted by hitrate) ==="
50+
sort -k2 -rn "$RESULTS_FILE"
51+
52+
# Find the best
53+
best_line=$(sort -k2 -rn "$RESULTS_FILE" | head -1)
54+
best_freq=$(echo "$best_line" | awk '{print $1}')
55+
best_hitrate=$(echo "$best_line" | awk '{print $2}')
56+
57+
echo ""
58+
echo "=== OPTIMAL for ${SIZE}K ==="
59+
echo "maxFreq = $best_freq with hitrate=$best_hitrate%"
60+
61+
# Restore original value
62+
sed -i '' "s/maxFreq = [0-9]*/maxFreq = 5/" "$S3FIFO"
63+
echo ""
64+
echo "Restored maxFreq to 5"

‎benchmarks/tune_peak_freq.sh‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
#!/bin/bash
2+
# Binary search for optimal maxPeakFreq across cache sizes
3+
set -e
4+
5+
S3FIFO="$(dirname "$0")/../s3fifo.go"
6+
SIZE="${1:-16}"
7+
RESULTS_FILE="/tmp/peak_freq_${SIZE}k.txt"
8+
9+
# Clear previous results
10+
> "$RESULTS_FILE"
11+
12+
run_benchmark() {
13+
local freq=$1
14+
15+
# Skip if already tested
16+
if grep -q "^$freq " "$RESULTS_FILE" 2>/dev/null; then
17+
return 0
18+
fi
19+
20+
# Update maxPeakFreq in s3fifo.go
21+
sed -i '' "s/maxPeakFreq = [0-9]*/maxPeakFreq = $freq/" "$S3FIFO"
22+
23+
# Run benchmark - capture output
24+
output=$(SIZES=$SIZE SUITES=hitrate make benchmark 2>&1) || true
25+
26+
# Extract the average from Category Summary line for multicache
27+
avg=$(echo "$output" | grep -E '^\s*\| multicache' | tail -1 | awk -F'|' '{print $3}' | grep -o '[0-9.]*')
28+
29+
if [ -z "$avg" ]; then
30+
echo "peakFreq=$freq: FAILED"
31+
return 1
32+
fi
33+
34+
echo "maxPeakFreq=$freq: hitrate=$avg%"
35+
echo "$freq $avg" >> "$RESULTS_FILE"
36+
}
37+
38+
echo "=== Tuning maxPeakFreq for ${SIZE}K cache ==="
39+
echo ""
40+
41+
# Test range from 5 to 40 (current is 21)
42+
echo "=== Coarse scan 5-40 ==="
43+
for freq in 5 10 15 21 25 30 35 40; do
44+
run_benchmark $freq
45+
done
46+
47+
echo ""
48+
echo "=== Results So Far ==="
49+
sort -k2 -rn "$RESULTS_FILE" | head -10
50+
51+
# Find best from initial scan
52+
best_line=$(sort -k2 -rn "$RESULTS_FILE" | head -1)
53+
best_freq=$(echo "$best_line" | awk '{print $1}')
54+
55+
echo ""
56+
echo "Best so far: $best_freq"
57+
58+
# Fine-tune around best
59+
echo ""
60+
echo "=== Fine-tuning around $best_freq ==="
61+
low=$((best_freq - 8))
62+
high=$((best_freq + 8))
63+
[ $low -lt 3 ] && low=3
64+
65+
for freq in $(seq $low 2 $high); do
66+
run_benchmark $freq
67+
done
68+
69+
echo ""
70+
echo "=== Final Results (sorted by hitrate) ==="
71+
sort -k2 -rn "$RESULTS_FILE" | head -15
72+
73+
# Find the best
74+
best_line=$(sort -k2 -rn "$RESULTS_FILE" | head -1)
75+
best_freq=$(echo "$best_line" | awk '{print $1}')
76+
best_hitrate=$(echo "$best_line" | awk '{print $2}')
77+
78+
echo ""
79+
echo "=== OPTIMAL for ${SIZE}K ==="
80+
echo "maxPeakFreq = $best_freq with hitrate=$best_hitrate%"
81+
82+
# Restore original value
83+
sed -i '' "s/maxPeakFreq = [0-9]*/maxPeakFreq = 21/" "$S3FIFO"
84+
echo ""
85+
echo "Restored maxPeakFreq to 21"

‎benchmarks/tune_small_ratio.sh‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
#!/bin/bash
2+
# Fine-grained binary search for optimal smallQueueRatio
3+
set -e
4+
5+
S3FIFO="$(dirname "$0")/../s3fifo.go"
6+
SIZE="${1:-16}"
7+
RESULTS_FILE="/tmp/small_ratio_${SIZE}k.txt"
8+
9+
# Clear previous results
10+
> "$RESULTS_FILE"
11+
12+
run_benchmark() {
13+
local ratio=$1
14+
15+
# Skip if already tested
16+
if grep -q "^$ratio " "$RESULTS_FILE" 2>/dev/null; then
17+
return 0
18+
fi
19+
20+
# Update smallQueueRatio in s3fifo.go
21+
sed -i '' "s/smallQueueRatio = [0-9]*/smallQueueRatio = $ratio/" "$S3FIFO"
22+
23+
# Run benchmark - capture output
24+
output=$(SIZES=$SIZE SUITES=hitrate make benchmark 2>&1) || true
25+
26+
# Extract the average from Category Summary line for multicache
27+
avg=$(echo "$output" | grep -E '^\s*\| multicache' | tail -1 | awk -F'|' '{print $3}' | grep -o '[0-9.]*')
28+
29+
if [ -z "$avg" ]; then
30+
echo "ratio=$ratio: FAILED"
31+
return 1
32+
fi
33+
34+
echo "ratio=$ratio ($(echo "scale=1; $ratio/10" | bc)%): hitrate=$avg%"
35+
echo "$ratio $avg" >> "$RESULTS_FILE"
36+
}
37+
38+
echo "=== Fine-tuning smallQueueRatio for ${SIZE}K cache ==="
39+
echo ""
40+
41+
# Binary search between bounds
42+
low=100
43+
high=160
44+
iterations=0
45+
max_iterations=20
46+
47+
while [ $((high - low)) -gt 2 ] && [ $iterations -lt $max_iterations ]; do
48+
mid=$(( (low + high) / 2 ))
49+
left=$(( (low + mid) / 2 ))
50+
right=$(( (mid + high) / 2 ))
51+
52+
echo "--- Iteration $((iterations+1)): testing $left, $mid, $right (range: $low-$high) ---"
53+
54+
run_benchmark $left
55+
run_benchmark $mid
56+
run_benchmark $right
57+
58+
# Get results
59+
left_val=$(grep "^$left " "$RESULTS_FILE" | awk '{print $2}')
60+
mid_val=$(grep "^$mid " "$RESULTS_FILE" | awk '{print $2}')
61+
right_val=$(grep "^$right " "$RESULTS_FILE" | awk '{print $2}')
62+
63+
# Find best and narrow range
64+
best_val=$left_val
65+
best_pos="left"
66+
67+
if [ "$(echo "$mid_val > $best_val" | bc -l)" -eq 1 ]; then
68+
best_val=$mid_val
69+
best_pos="mid"
70+
fi
71+
if [ "$(echo "$right_val > $best_val" | bc -l)" -eq 1 ]; then
72+
best_val=$right_val
73+
best_pos="right"
74+
fi
75+
76+
case $best_pos in
77+
left) high=$mid ;;
78+
mid) low=$left; high=$right ;;
79+
right) low=$mid ;;
80+
esac
81+
82+
iterations=$((iterations + 1))
83+
done
84+
85+
# Final sweep of remaining range
86+
echo ""
87+
echo "--- Final sweep: $low to $high ---"
88+
for ratio in $(seq $low $high); do
89+
run_benchmark $ratio
90+
done
91+
92+
echo ""
93+
echo "=== Results for ${SIZE}K (sorted by hitrate) ==="
94+
sort -k2 -rn "$RESULTS_FILE" | head -10
95+
96+
# Find the best
97+
best_line=$(sort -k2 -rn "$RESULTS_FILE" | head -1)
98+
best_ratio=$(echo "$best_line" | awk '{print $1}')
99+
best_hitrate=$(echo "$best_line" | awk '{print $2}')
100+
101+
echo ""
102+
echo "=== OPTIMAL for ${SIZE}K ==="
103+
echo "smallQueueRatio = $best_ratio ($(echo "scale=1; $best_ratio/10" | bc)%) with hitrate=$best_hitrate%"
104+
105+
# Restore original value
106+
sed -i '' "s/smallQueueRatio = [0-9]*/smallQueueRatio = 137/" "$S3FIFO"
107+
echo ""
108+
echo "Restored smallQueueRatio to 137"

‎pkg/store/cloudrun/go.mod‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,9 @@ module github.com/codeGROOVE-dev/fido/pkg/store/cloudrun
33
go 1.25.4
44

55
require (
6-
github.com/codeGROOVE-dev/fido/pkg/store/compress v1.8.0
7-
github.com/codeGROOVE-dev/fido/pkg/store/datastore v1.8.0
8-
github.com/codeGROOVE-dev/fido/pkg/store/localfs v1.8.0
6+
github.com/codeGROOVE-dev/fido/pkg/store/compress v1.9.0
7+
github.com/codeGROOVE-dev/fido/pkg/store/datastore v1.9.0
8+
github.com/codeGROOVE-dev/fido/pkg/store/localfs v1.9.0
99
)
1010

1111
require (

‎pkg/store/datastore/go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ go 1.25.4
44

55
require (
66
github.com/codeGROOVE-dev/ds9 v0.8.0
7-
github.com/codeGROOVE-dev/fido/pkg/store/compress v1.8.0
7+
github.com/codeGROOVE-dev/fido/pkg/store/compress v1.9.0
88
)
99

1010
require github.com/klauspost/compress v1.18.2 // indirect

‎pkg/store/localfs/go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ module github.com/codeGROOVE-dev/fido/pkg/store/localfs
33
go 1.25.4
44

55
require (
6-
github.com/codeGROOVE-dev/fido/pkg/store/compress v1.8.0
6+
github.com/codeGROOVE-dev/fido/pkg/store/compress v1.9.0
77
github.com/klauspost/compress v1.18.2
88
github.com/pierrec/lz4/v4 v4.1.22
99
)

‎pkg/store/null/go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ module github.com/codeGROOVE-dev/fido/pkg/store/null
22

33
go 1.25.4
44

5-
require github.com/codeGROOVE-dev/fido/pkg/store/compress v1.8.0
5+
require github.com/codeGROOVE-dev/fido/pkg/store/compress v1.9.0
66

77
require github.com/klauspost/compress v1.18.2 // indirect
88

‎pkg/store/valkey/go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ module github.com/codeGROOVE-dev/fido/pkg/store/valkey
33
go 1.25.4
44

55
require (
6-
github.com/codeGROOVE-dev/fido/pkg/store/compress v1.8.0
6+
github.com/codeGROOVE-dev/fido/pkg/store/compress v1.9.0
77
github.com/valkey-io/valkey-go v1.0.70
88
)
99

0 commit comments

Comments
 (0)