ghost ratio experiments - 0.5 is a sweet spot

Author: Thomas Stromberg

memory.go

@@ -32,7 +32,7 @@ func Memory[K comparable, V any](opts ...Option) *MemoryCache[K, V] {
 	}
 
 	return &MemoryCache[K, V]{
-		memory:     newS3FIFO[K, V](cfg.size),
+		memory:     newS3FIFO[K, V](cfg),
 		defaultTTL: cfg.defaultTTL,
 	}
 }
@@ -133,11 +133,15 @@ type config struct {
 	size       int
 	defaultTTL time.Duration
 	warmup     int
+	smallRatio float64
+	ghostRatio float64
 }
 
 func defaultConfig() *config {
 	return &config{
-		size: 16384, // 2^14, divides evenly by numShards
+		size:       16384, // 2^14, divides evenly by numShards
+		smallRatio: 0.1,   // 10% small queue
+		ghostRatio: 0.5,   // 50% ghost queue
 	}
 }
 
@@ -151,6 +155,22 @@ func WithSize(n int) Option {
 	}
 }
 
+// WithSmallRatio sets the ratio of the small queue to the total cache size.
+// Default is 0.1 (10%).
+func WithSmallRatio(r float64) Option {
+	return func(c *config) {
+		c.smallRatio = r
+	}
+}
+
+// WithGhostRatio sets the ratio of the ghost queue to the total cache size.
+// Default is 1.0 (100%).
+func WithGhostRatio(r float64) Option {
+	return func(c *config) {
+		c.ghostRatio = r
+	}
+}
+
 // WithTTL sets the default TTL for cache items.
 // Items without an explicit TTL will use this value.
 func WithTTL(d time.Duration) Option {

persistent.go

@@ -48,7 +48,7 @@ func Persistent[K comparable, V any](ctx context.Context, p PersistenceLayer[K,
 
 	cache := &PersistentCache[K, V]{
 		Store:      p,
-		memory:     newS3FIFO[K, V](cfg.size),
+		memory:     newS3FIFO[K, V](cfg),
 		defaultTTL: cfg.defaultTTL,
 		warmup:     cfg.warmup,
 	}

s3fifo.go

@@ -222,7 +222,8 @@ type entry[K comparable, V any] struct {
 }
 
 // newS3FIFO creates a new sharded S3-FIFO cache with the given total capacity.
-func newS3FIFO[K comparable, V any](capacity int) *s3fifo[K, V] {
+func newS3FIFO[K comparable, V any](cfg *config) *s3fifo[K, V] {
+	capacity := cfg.size
 	if capacity <= 0 {
 		capacity = 16384 // 2^14, divides evenly by 16 shards
 	}
@@ -263,22 +264,22 @@ func newS3FIFO[K comparable, V any](capacity int) *s3fifo[K, V] {
 	}
 
 	for i := range numShards {
-		c.shards[i] = newShard[K, V](shardCap)
+		c.shards[i] = newShard[K, V](shardCap, cfg.smallRatio, cfg.ghostRatio)
 	}
 
 	return c
 }
 
 // newShard creates a new S3-FIFO shard with the given capacity.
-func newShard[K comparable, V any](capacity int) *shard[K, V] {
-	// Small queue: 10% of cache capacity (S3-FIFO paper recommendation).
-	smallCap := capacity / 10
+func newShard[K comparable, V any](capacity int, smallRatio, ghostRatio float64) *shard[K, V] {
+	// Small queue: recommended 10%
+	smallCap := int(float64(capacity) * smallRatio)
 	if smallCap < 1 {
 		smallCap = 1
 	}
 
-	// Ghost queue: 100% of cache capacity to maximize re-admission detection.
-	ghostCap := capacity
+	// Ghost queue: recommended 100%
+	ghostCap := int(float64(capacity) * ghostRatio)
 	if ghostCap < 1 {
 		ghostCap = 1
 	}
@@ -421,20 +422,17 @@ func (s *shard[K, V]) getOrSet(key K, value V, expiryNano int64) (V, bool) {
 	// Fast path: try read lock first (optimistic for cache hits)
 	s.mu.RLock()
 	if ent, ok := s.items[key]; ok {
-		// Check expiration
-		if ent.expiryNano != 0 && time.Now().UnixNano() > ent.expiryNano {
-			// Expired - need write lock to update
-			s.mu.RUnlock()
-			// Fall through to slow path
-		} else {
-			// Hit - mark as accessed and return
+		// Check if NOT expired - return early with cache hit
+		if ent.expiryNano == 0 || time.Now().UnixNano() <= ent.expiryNano {
 			if !ent.accessed.Load() {
 				ent.accessed.Store(true)
 			}
 			v := ent.value
 			s.mu.RUnlock()
 			return v, true
 		}
+		// Expired - need write lock to update, fall through to slow path
+		s.mu.RUnlock()
 	} else {
 		s.mu.RUnlock()
 	}

s3fifo_test.go

@@ -8,7 +8,7 @@ import (
 )
 
 func TestS3FIFO_BasicOperations(t *testing.T) {
-	cache := newS3FIFO[string, int](100)
+	cache := newS3FIFO[string, int](&config{size: 100, smallRatio: 0.1, ghostRatio: 1.0})
 
 	// Test set and get
 	cache.setToMemory("key1", 42, 0)
@@ -35,7 +35,7 @@ func TestS3FIFO_BasicOperations(t *testing.T) {
 }
 
 func TestS3FIFO_Capacity(t *testing.T) {
-	cache := newS3FIFO[int, string](20000)
+	cache := newS3FIFO[int, string](&config{size: 20000, smallRatio: 0.1, ghostRatio: 1.0})
 
 	// Fill cache well beyond capacity
 	for i := range 30000 {
@@ -65,7 +65,7 @@ func TestS3FIFO_CapacityAccuracy(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(fmt.Sprintf("capacity_%d", tc.requested), func(t *testing.T) {
-			cache := newS3FIFO[int, int](tc.requested)
+			cache := newS3FIFO[int, int](&config{size: tc.requested, smallRatio: 0.1, ghostRatio: 1.0})
 
 			// Insert many more items than capacity
 			for i := range tc.requested * 3 {
@@ -88,7 +88,7 @@ func TestS3FIFO_CapacityAccuracy(t *testing.T) {
 }
 
 func TestS3FIFO_Eviction(t *testing.T) {
-	cache := newS3FIFO[int, int](10000)
+	cache := newS3FIFO[int, int](&config{size: 10000, smallRatio: 0.1, ghostRatio: 1.0})
 
 	// Fill cache to capacity
 	for i := range 10000 {
@@ -115,7 +115,7 @@ func TestS3FIFO_Eviction(t *testing.T) {
 }
 
 func TestS3FIFO_GhostQueue(t *testing.T) {
-	cache := newS3FIFO[string, int](12) // 3 per shard
+	cache := newS3FIFO[string, int](&config{size: 12, smallRatio: 0.1, ghostRatio: 1.0})
 
 	// Fill one shard's worth
 	cache.setToMemory("a", 1, 0)
@@ -135,7 +135,7 @@ func TestS3FIFO_GhostQueue(t *testing.T) {
 }
 
 func TestS3FIFO_TTL(t *testing.T) {
-	cache := newS3FIFO[string, int](10)
+	cache := newS3FIFO[string, int](&config{size: 10, smallRatio: 0.1, ghostRatio: 1.0})
 
 	// Set item with past expiry
 	past := time.Now().Add(-1 * time.Second).UnixNano()
@@ -157,7 +157,7 @@ func TestS3FIFO_TTL(t *testing.T) {
 }
 
 func TestS3FIFO_Concurrent(t *testing.T) {
-	cache := newS3FIFO[int, int](1000)
+	cache := newS3FIFO[int, int](&config{size: 1000, smallRatio: 0.1, ghostRatio: 1.0})
 	var wg sync.WaitGroup
 
 	// Concurrent writers
@@ -193,7 +193,7 @@ func TestS3FIFO_Concurrent(t *testing.T) {
 func TestS3FIFO_FrequencyPromotion(t *testing.T) {
 	// Use a larger capacity to ensure meaningful per-shard capacity
 	// With 512 shards, 10000 items = ~20 per shard
-	cache := newS3FIFO[int, int](10000)
+	cache := newS3FIFO[int, int](&config{size: 10000, smallRatio: 0.1, ghostRatio: 1.0})
 
 	// Fill cache with items using int keys for predictable sharding
 	for i := range 10000 {
@@ -233,7 +233,7 @@ func TestS3FIFO_FrequencyPromotion(t *testing.T) {
 
 func TestS3FIFO_SmallCapacity(t *testing.T) {
 	// Test with capacity of 12 (3 per shard)
-	cache := newS3FIFO[string, int](12)
+	cache := newS3FIFO[string, int](&config{size: 12, smallRatio: 0.1, ghostRatio: 1.0})
 
 	// Fill to capacity
 	cache.setToMemory("a", 1, 0)
@@ -259,7 +259,7 @@ func TestS3FIFO_SmallCapacity(t *testing.T) {
 }
 
 func BenchmarkS3FIFO_Set(b *testing.B) {
-	cache := newS3FIFO[int, int](10000)
+	cache := newS3FIFO[int, int](&config{size: 10000, smallRatio: 0.1, ghostRatio: 1.0})
 	b.ResetTimer()
 
 	for i := range b.N {
@@ -268,7 +268,7 @@ func BenchmarkS3FIFO_Set(b *testing.B) {
 }
 
 func BenchmarkS3FIFO_Get(b *testing.B) {
-	cache := newS3FIFO[int, int](10000)
+	cache := newS3FIFO[int, int](&config{size: 10000, smallRatio: 0.1, ghostRatio: 1.0})
 	for i := range 10000 {
 		cache.setToMemory(i, i, 0)
 	}
@@ -280,7 +280,7 @@ func BenchmarkS3FIFO_Get(b *testing.B) {
 }
 
 func BenchmarkS3FIFO_GetParallel(b *testing.B) {
-	cache := newS3FIFO[int, int](10000)
+	cache := newS3FIFO[int, int](&config{size: 10000, smallRatio: 0.1, ghostRatio: 1.0})
 	for i := range 10000 {
 		cache.setToMemory(i, i, 0)
 	}
@@ -296,7 +296,7 @@ func BenchmarkS3FIFO_GetParallel(b *testing.B) {
 }
 
 func BenchmarkS3FIFO_Mixed(b *testing.B) {
-	cache := newS3FIFO[int, int](10000)
+	cache := newS3FIFO[int, int](&config{size: 10000, smallRatio: 0.1, ghostRatio: 1.0})
 	b.ResetTimer()
 
 	for i := range b.N {
@@ -415,7 +415,7 @@ func TestS3FIFODetailed(t *testing.T) {
 
 func TestS3FIFO_Flush(t *testing.T) {
 	// Use int keys for predictable sharding, large capacity to avoid evictions
-	cache := newS3FIFO[int, int](10000)
+	cache := newS3FIFO[int, int](&config{size: 10000, smallRatio: 0.1, ghostRatio: 1.0})
 
 	// Add some items (fewer than capacity to avoid eviction)
 	for i := range 100 {
@@ -457,7 +457,7 @@ func TestS3FIFO_Flush(t *testing.T) {
 }
 
 func TestS3FIFO_FlushEmpty(t *testing.T) {
-	cache := newS3FIFO[string, int](100)
+	cache := newS3FIFO[string, int](&config{size: 100, smallRatio: 0.1, ghostRatio: 1.0})
 
 	// Flush empty cache
 	removed := cache.flushMemory()
@@ -487,7 +487,7 @@ func TestS3FIFO_VariousKeyTypes(t *testing.T) {
 	// This exercises different code paths in getShard/shardIndexSlow.
 
 	t.Run("int", func(t *testing.T) {
-		cache := newS3FIFO[int, string](100)
+		cache := newS3FIFO[int, string](&config{size: 100, smallRatio: 0.1, ghostRatio: 1.0})
 		cache.setToMemory(42, "forty-two", 0)
 		cache.setToMemory(-1, "negative", 0)
 		cache.setToMemory(0, "zero", 0)
@@ -504,7 +504,7 @@ func TestS3FIFO_VariousKeyTypes(t *testing.T) {
 	})
 
 	t.Run("int64", func(t *testing.T) {
-		cache := newS3FIFO[int64, string](100)
+		cache := newS3FIFO[int64, string](&config{size: 100, smallRatio: 0.1, ghostRatio: 1.0})
 		cache.setToMemory(int64(1<<62), "large", 0)
 		cache.setToMemory(int64(-1), "negative", 0)
 
@@ -517,7 +517,7 @@ func TestS3FIFO_VariousKeyTypes(t *testing.T) {
 	})
 
 	t.Run("uint", func(t *testing.T) {
-		cache := newS3FIFO[uint, string](100)
+		cache := newS3FIFO[uint, string](&config{size: 100, smallRatio: 0.1, ghostRatio: 1.0})
 		cache.setToMemory(uint(0), "zero", 0)
 		cache.setToMemory(uint(100), "hundred", 0)
 
@@ -531,7 +531,7 @@ func TestS3FIFO_VariousKeyTypes(t *testing.T) {
 
 	t.Run("uint64", func(t *testing.T) {
 		// Use larger size to ensure per-shard capacity > 1 (2048 shards)
-		cache := newS3FIFO[uint64, string](10000)
+		cache := newS3FIFO[uint64, string](&config{size: 10000, smallRatio: 0.1, ghostRatio: 1.0})
 		cache.setToMemory(uint64(1<<63), "large", 0)
 		cache.setToMemory(uint64(0), "zero", 0)
 
@@ -544,7 +544,7 @@ func TestS3FIFO_VariousKeyTypes(t *testing.T) {
 	})
 
 	t.Run("string", func(t *testing.T) {
-		cache := newS3FIFO[string, int](100)
+		cache := newS3FIFO[string, int](&config{size: 100, smallRatio: 0.1, ghostRatio: 1.0})
 		cache.setToMemory("hello", 1, 0)
 		cache.setToMemory("", 2, 0) // empty string is valid
 		unicode := "unicode-\u65e5\u672c\u8a9e"
@@ -563,7 +563,7 @@ func TestS3FIFO_VariousKeyTypes(t *testing.T) {
 
 	t.Run("fmt.Stringer", func(t *testing.T) {
 		// Tests the fmt.Stringer fast path in shardIndexSlow
-		cache := newS3FIFO[stringerKey, string](100)
+		cache := newS3FIFO[stringerKey, string](&config{size: 100, smallRatio: 0.1, ghostRatio: 1.0})
 		k1 := stringerKey{id: 1}
 		k2 := stringerKey{id: 2}
 		k3 := stringerKey{id: 999}
@@ -592,7 +592,7 @@ func TestS3FIFO_VariousKeyTypes(t *testing.T) {
 	t.Run("plain struct", func(t *testing.T) {
 		// Tests the fmt.Sprintf fallback in shardIndexSlow.
 		// This is not fast, but should be reliable.
-		cache := newS3FIFO[plainKey, string](100)
+		cache := newS3FIFO[plainKey, string](&config{size: 100, smallRatio: 0.1, ghostRatio: 1.0})
 		k1 := plainKey{a: 1, b: "one"}
 		k2 := plainKey{a: 2, b: "two"}
 		k3 := plainKey{a: 1, b: "one"} // Same as k1