feat: add basic cache eviction policy: LRU/LFU/FIFO (#166)

* feat: add basic cache eviction policy: LRU/LFU/FIFO

Signed-off-by: Alex Wang <[email protected]>

* use EvictionPolicyType

Signed-off-by: Alex Wang <[email protected]>

* update doc

Signed-off-by: Alex Wang <[email protected]>

---------

Signed-off-by: Alex Wang <[email protected]>
Co-authored-by: Huamin Chen <[email protected]>

Author: aeft

config/config.yaml

@@ -8,6 +8,7 @@ semantic_cache:
   similarity_threshold: 0.8
   max_entries: 1000  # Only applies to memory backend
   ttl_seconds: 3600
+  eviction_policy: "fifo"  # "fifo", "lru", "lfu", currently only supports memory backend
 
   # For production environments, use Milvus for scalable caching:
   # backend_type: "milvus"

src/semantic-router/pkg/cache/cache_factory.go

@@ -27,6 +27,7 @@ func NewCacheBackend(config CacheConfig) (CacheBackend, error) {
 			SimilarityThreshold: config.SimilarityThreshold,
 			MaxEntries:          config.MaxEntries,
 			TTLSeconds:          config.TTLSeconds,
+			EvictionPolicy:      config.EvictionPolicy,
 		}
 		return NewInMemoryCache(options), nil
 

src/semantic-router/pkg/cache/cache_interface.go

@@ -10,7 +10,9 @@ type CacheEntry struct {
 	Model        string
 	Query        string
 	Embedding    []float32
-	Timestamp    time.Time
+	Timestamp    time.Time // Creation time (when the entry was added or completed with a response)
+	LastAccessAt time.Time // Last access time
+	HitCount     int64     // Access count
 }
 
 // CacheBackend defines the interface for semantic cache implementations
@@ -58,6 +60,20 @@ const (
 	MilvusCacheType CacheBackendType = "milvus"
 )
 
+// EvictionPolicyType defines the available eviction policies
+type EvictionPolicyType string
+
+const (
+	// FIFOEvictionPolicyType specifies the FIFO eviction policy
+	FIFOEvictionPolicyType EvictionPolicyType = "fifo"
+
+	// LRUEvictionPolicyType specifies the LRU eviction policy
+	LRUEvictionPolicyType EvictionPolicyType = "lru"
+
+	// LFUEvictionPolicyType specifies the LFU eviction policy
+	LFUEvictionPolicyType EvictionPolicyType = "lfu"
+)
+
 // CacheConfig contains configuration settings shared across all cache backends
 type CacheConfig struct {
 	// BackendType specifies which cache implementation to use
@@ -75,6 +91,9 @@ type CacheConfig struct {
 	// TTLSeconds sets cache entry expiration time (0 disables expiration)
 	TTLSeconds int `yaml:"ttl_seconds,omitempty"`
 
+	// EvictionPolicy defines the eviction policy for in-memory cache ("fifo", "lru", "lfu")
+	EvictionPolicy EvictionPolicyType `yaml:"eviction_policy,omitempty"`
+
 	// BackendConfigPath points to backend-specific configuration files
 	BackendConfigPath string `yaml:"backend_config_path,omitempty"`
 }

src/semantic-router/pkg/cache/eviction_policy.go

@@ -0,0 +1,58 @@
+package cache
+
+type EvictionPolicy interface {
+	SelectVictim(entries []CacheEntry) int
+}
+
+type FIFOPolicy struct{}
+
+func (p *FIFOPolicy) SelectVictim(entries []CacheEntry) int {
+	if len(entries) == 0 {
+		return -1
+	}
+
+	oldestIdx := 0
+	for i := 1; i < len(entries); i++ {
+		if entries[i].Timestamp.Before(entries[oldestIdx].Timestamp) {
+			oldestIdx = i
+		}
+	}
+	return oldestIdx
+}
+
+type LRUPolicy struct{}
+
+func (p *LRUPolicy) SelectVictim(entries []CacheEntry) int {
+	if len(entries) == 0 {
+		return -1
+	}
+
+	oldestIdx := 0
+	for i := 1; i < len(entries); i++ {
+		if entries[i].LastAccessAt.Before(entries[oldestIdx].LastAccessAt) {
+			oldestIdx = i
+		}
+	}
+	return oldestIdx
+}
+
+type LFUPolicy struct{}
+
+func (p *LFUPolicy) SelectVictim(entries []CacheEntry) int {
+	if len(entries) == 0 {
+		return -1
+	}
+
+	victimIdx := 0
+	for i := 1; i < len(entries); i++ {
+		if entries[i].HitCount < entries[victimIdx].HitCount {
+			victimIdx = i
+		} else if entries[i].HitCount == entries[victimIdx].HitCount {
+			// Use LRU as tiebreaker to avoid random selection
+			if entries[i].LastAccessAt.Before(entries[victimIdx].LastAccessAt) {
+				victimIdx = i
+			}
+		}
+	}
+	return victimIdx
+}

src/semantic-router/pkg/cache/eviction_policy_test.go

@@ -0,0 +1,89 @@
+package cache
+
+import (
+	"testing"
+	"time"
+)
+
+func TestFIFOPolicy(t *testing.T) {
+	policy := &FIFOPolicy{}
+
+	// Test empty entries
+	if victim := policy.SelectVictim([]CacheEntry{}); victim != -1 {
+		t.Errorf("Expected -1 for empty entries, got %d", victim)
+	}
+
+	// Test with entries
+	now := time.Now()
+	entries := []CacheEntry{
+		{Query: "query1", Timestamp: now.Add(-3 * time.Second)},
+		{Query: "query2", Timestamp: now.Add(-1 * time.Second)},
+		{Query: "query3", Timestamp: now.Add(-2 * time.Second)},
+	}
+
+	victim := policy.SelectVictim(entries)
+	if victim != 0 {
+		t.Errorf("Expected victim index 0 (oldest), got %d", victim)
+	}
+}
+
+func TestLRUPolicy(t *testing.T) {
+	policy := &LRUPolicy{}
+
+	// Test empty entries
+	if victim := policy.SelectVictim([]CacheEntry{}); victim != -1 {
+		t.Errorf("Expected -1 for empty entries, got %d", victim)
+	}
+
+	// Test with entries
+	now := time.Now()
+	entries := []CacheEntry{
+		{Query: "query1", LastAccessAt: now.Add(-3 * time.Second)},
+		{Query: "query2", LastAccessAt: now.Add(-1 * time.Second)},
+		{Query: "query3", LastAccessAt: now.Add(-2 * time.Second)},
+	}
+
+	victim := policy.SelectVictim(entries)
+	if victim != 0 {
+		t.Errorf("Expected victim index 0 (least recently used), got %d", victim)
+	}
+}
+
+func TestLFUPolicy(t *testing.T) {
+	policy := &LFUPolicy{}
+
+	// Test empty entries
+	if victim := policy.SelectVictim([]CacheEntry{}); victim != -1 {
+		t.Errorf("Expected -1 for empty entries, got %d", victim)
+	}
+
+	// Test with entries
+	now := time.Now()
+	entries := []CacheEntry{
+		{Query: "query1", HitCount: 5, LastAccessAt: now.Add(-2 * time.Second)},
+		{Query: "query2", HitCount: 1, LastAccessAt: now.Add(-3 * time.Second)},
+		{Query: "query3", HitCount: 3, LastAccessAt: now.Add(-1 * time.Second)},
+	}
+
+	victim := policy.SelectVictim(entries)
+	if victim != 1 {
+		t.Errorf("Expected victim index 1 (least frequently used), got %d", victim)
+	}
+}
+
+func TestLFUPolicyTiebreaker(t *testing.T) {
+	policy := &LFUPolicy{}
+
+	// Test tiebreaker: same frequency, choose least recently used
+	now := time.Now()
+	entries := []CacheEntry{
+		{Query: "query1", HitCount: 2, LastAccessAt: now.Add(-1 * time.Second)},
+		{Query: "query2", HitCount: 2, LastAccessAt: now.Add(-3 * time.Second)},
+		{Query: "query3", HitCount: 5, LastAccessAt: now.Add(-2 * time.Second)},
+	}
+
+	victim := policy.SelectVictim(entries)
+	if victim != 1 {
+		t.Errorf("Expected victim index 1 (LRU tiebreaker), got %d", victim)
+	}
+}