HNSW entries disappear after eviction in the in-memory cache

[cryo-zd]

Reproduce:

It("should keep existing HNSW nodes searchable after eviction", func() {
			cacheWithHNSW := NewInMemoryCache(InMemoryCacheOptions{
				Enabled:             true,
				SimilarityThreshold: 0.1,
				MaxEntries:          2,
				TTLSeconds:          60, // Set TTL long enough to avoid expiration during test
				EvictionPolicy:      FIFOEvictionPolicyType,
				UseHNSW:             true,
				HNSWM:               4,
				HNSWEfConstruction:  8,
				HNSWEfSearch:        8,
				EmbeddingModel:      "bert",
			})
			defer cacheWithHNSW.Close()

			err := cacheWithHNSW.AddEntry("req-1", "test-model", "first query text", []byte("request-1"), []byte("response-1"))
			Expect(err).NotTo(HaveOccurred())

			err = cacheWithHNSW.AddEntry("req-2", "test-model", "second query text", []byte("request-2"), []byte("response-2"))
			Expect(err).NotTo(HaveOccurred())

			// Sanity check: the second entry should be retrievable before any eviction occurs.
			resp, found, err := cacheWithHNSW.FindSimilar("test-model", "second query text")
			Expect(err).NotTo(HaveOccurred())
			Expect(found).To(BeTrue())
			Expect(resp).To(Equal([]byte("response-2")))

			// Adding a third entry triggers eviction (max entries = 2).
			err = cacheWithHNSW.AddEntry("req-3", "test-model", "third query text", []byte("request-3"), []byte("response-3"))
			Expect(err).NotTo(HaveOccurred())

			// Entry 2 should still be searchable even after eviction reshuffles the slice.
			resp, found, err = cacheWithHNSW.FindSimilar("test-model", "second query text")
			Expect(err).NotTo(HaveOccurred())
			Expect(found).To(BeTrue())
			Expect(resp).To(Equal([]byte("response-2")))
		})

Related

• cache only rebuild HNSW in func (c *InMemoryCache) cleanupExpiredEntries():
  

  semantic-router/src/semantic-router/pkg/cache/inmemory_cache.go

  Lines 574 to 579 in ee6e87e

  	// Rebuild HNSW index if entries were removed
  	if expiredCount > 0 && c.useHNSW && c.hnswIndex != nil {
  	c.rebuildHNSWIndex()
  	}

• Eviction:
  

  semantic-router/src/semantic-router/pkg/cache/inmemory_cache.go

  Lines 625 to 632 in ee6e87e

  	// If using HNSW, we need to rebuild the index after eviction
  	// For simplicity, we'll mark that a rebuild is needed
  	if c.useHNSW && c.hnswIndex != nil {
  	// Remove the node from HNSW index
  	// Note: HNSW doesn't support efficient deletion, so we'll rebuild on next search if needed
  	c.hnswIndex.markStale()
  	}