Fix storeTag regex performance bottleneck

pkg/surrogate/providers/common.go

@@ -162,11 +162,37 @@ func (s *baseStorage) init(config configurationtypes.AbstractConfigurationInterf
 	s.duration = storageToInfiniteTTLMap[s.Storage.Name()]
 }
 
-func (s *baseStorage) storeTag(tag string, cacheKey string, re *regexp.Regexp) {
+// containsCacheKey checks if the cacheKey already exists in the comma-separated currentValue.
+// This is much faster than regex matching, especially for long strings.
+func containsCacheKey(currentValue, cacheKey string) bool {
+	if currentValue == "" {
+		return false
+	}
+	// Check for exact match at various positions:
+	// 1. Exact match of entire string
+	if currentValue == cacheKey {
+		return true
+	}
+	// 2. At the beginning: "cacheKey,"
+	if strings.HasPrefix(currentValue, cacheKey+souinStorageSeparator) {
+		return true
+	}
+	// 3. At the end: ",cacheKey"
+	if strings.HasSuffix(currentValue, souinStorageSeparator+cacheKey) {
+		return true
+	}
+	// 4. In the middle: ",cacheKey,"
+	if strings.Contains(currentValue, souinStorageSeparator+cacheKey+souinStorageSeparator) {
+		return true
+	}
+	return false
+}
+
+func (s *baseStorage) storeTag(tag string, cacheKey string) {
 	defer s.mu.Unlock()
 	s.mu.Lock()
 	currentValue := string(s.Storage.Get(surrogatePrefix + tag))
-	if !re.MatchString(currentValue) {
+	if !containsCacheKey(currentValue, cacheKey) {
 		s.logger.Debugf("Store the tag %s", tag)
 		_ = s.Storage.Set(surrogatePrefix+tag, []byte(currentValue+souinStorageSeparator+cacheKey), s.duration)
 	}
@@ -222,29 +248,28 @@ func (s *baseStorage) Store(response *http.Response, cacheKey, uri string) error
 
 	cacheKey = url.QueryEscape(cacheKey)
 
-	urlRegexp := regexp.MustCompile("(^|" + regexp.QuoteMeta(souinStorageSeparator) + ")" + regexp.QuoteMeta(cacheKey) + "(" + regexp.QuoteMeta(souinStorageSeparator) + "|$)")
 	keys := s.ParseHeaders(s.parent.getSurrogateKey(h))
 
 	for _, key := range keys {
 		_, v := s.parent.GetSurrogateControl(h)
 		if controls := s.ParseHeaders(v); len(controls) != 0 {
 			if len(controls) == 1 && controls[0] == "" {
-				s.storeTag(key, cacheKey, urlRegexp)
-				s.storeTag(uri, cacheKey, urlRegexp)
+				s.storeTag(key, cacheKey)
+				s.storeTag(uri, cacheKey)
 
 				continue
 			}
 			for _, control := range controls {
 				if s.parent.candidateStore(control) {
-					s.storeTag(key, cacheKey, urlRegexp)
-					s.storeTag(uri, cacheKey, urlRegexp)
+					s.storeTag(key, cacheKey)
+					s.storeTag(uri, cacheKey)
 
 					break
 				}
 			}
 		} else {
-			s.storeTag(key, cacheKey, urlRegexp)
-			s.storeTag(uri, cacheKey, urlRegexp)
+			s.storeTag(key, cacheKey)
+			s.storeTag(uri, cacheKey)
 		}
 	}
 

pkg/surrogate/providers/common_test.go

@@ -172,3 +172,39 @@ func TestBaseStorage_Store_Load(t *testing.T) {
 	// 	// t.Errorf("The surrogate storage should contain %d stored elements, %d given.", length+1, len(strings.Split(string(v), ",")))
 	// }
 }
+
+func TestContainsCacheKey(t *testing.T) {
+	testCases := []struct {
+		name         string
+		currentValue string
+		cacheKey     string
+		expected     bool
+	}{
+		{"empty current value", "", "key1", false},
+		{"exact match single key", "key1", "key1", true},
+		{"key at beginning", "key1,key2,key3", "key1", true},
+		{"key at end", "key1,key2,key3", "key3", true},
+		{"key in middle", "key1,key2,key3", "key2", true},
+		{"key not present", "key1,key2,key3", "key4", false},
+		{"partial match should not match", "key1,key2,key3", "key", false},
+		{"partial match at start should not match", "key12,key2,key3", "key1", false},
+		{"partial match at end should not match", "key1,key2,key34", "key3", false},
+		{"url encoded key present", "%2Fapi%2Fusers,other", "%2Fapi%2Fusers", true},
+		{"url encoded key not present", "%2Fapi%2Fusers,other", "%2Fapi%2Fposts", false},
+		{"key with special chars", "a%2Fb,c%2Fd", "a%2Fb", true},
+		{"empty key in empty value", "", "", false},
+		{"empty key in non-empty value", "key1,key2", "", false},
+		{"similar keys should not match", "product-123,product-1234", "product-12", false},
+		{"leading comma in stored value", ",key1,key2", "key1", true},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			result := containsCacheKey(tc.currentValue, tc.cacheKey)
+			if result != tc.expected {
+				t.Errorf("containsCacheKey(%q, %q) = %v, expected %v",
+					tc.currentValue, tc.cacheKey, result, tc.expected)
+			}
+		})
+	}
+}

pkg/surrogate/providers/types.go

@@ -2,7 +2,6 @@ package providers
 
 import (
 	"net/http"
-	"regexp"
 )
 
 // SurrogateInterface represents the interface to implement to be part
@@ -17,7 +16,7 @@ type SurrogateInterface interface {
 	Invalidate(method string, h http.Header)
 	purgeTag(string) []string
 	Store(*http.Response, string, string) error
-	storeTag(string, string, *regexp.Regexp)
+	storeTag(string, string)
 	ParseHeaders(string) []string
 	List() map[string]string
 	candidateStore(string) bool

plugins/traefik/override/surrogate/providers/common.go

@@ -120,11 +120,37 @@ func (s *baseStorage) init(config configurationtypes.AbstractConfigurationInterf
 	s.duration = storageToInfiniteTTLMap[s.Storage.Name()]
 }
 
-func (s *baseStorage) storeTag(tag string, cacheKey string, re *regexp.Regexp) {
+// containsCacheKey checks if the cacheKey already exists in the comma-separated currentValue.
+// This is much faster than regex matching, especially for long strings.
+func containsCacheKey(currentValue, cacheKey string) bool {
+	if currentValue == "" {
+		return false
+	}
+	// Check for exact match at various positions:
+	// 1. Exact match of entire string
+	if currentValue == cacheKey {
+		return true
+	}
+	// 2. At the beginning: "cacheKey,"
+	if strings.HasPrefix(currentValue, cacheKey+souinStorageSeparator) {
+		return true
+	}
+	// 3. At the end: ",cacheKey"
+	if strings.HasSuffix(currentValue, souinStorageSeparator+cacheKey) {
+		return true
+	}
+	// 4. In the middle: ",cacheKey,"
+	if strings.Contains(currentValue, souinStorageSeparator+cacheKey+souinStorageSeparator) {
+		return true
+	}
+	return false
+}
+
+func (s *baseStorage) storeTag(tag string, cacheKey string) {
 	defer s.mu.Unlock()
 	s.mu.Lock()
 	currentValue := string(s.Storage.Get(surrogatePrefix + tag))
-	if !re.MatchString(currentValue) {
+	if !containsCacheKey(currentValue, cacheKey) {
 		fmt.Printf("Store the tag %s", tag)
 		_ = s.Storage.Set(surrogatePrefix+tag, []byte(currentValue+souinStorageSeparator+cacheKey), -1)
 	}
@@ -195,29 +221,26 @@ func (s *baseStorage) Store(response *http.Response, cacheKey, uri string) error
 	cacheKey = url.QueryEscape(cacheKey)
 	staleKey := stalePrefix + cacheKey
 
-	urlRegexp := regexp.MustCompile("(^|" + regexp.QuoteMeta(souinStorageSeparator) + ")" + regexp.QuoteMeta(cacheKey) + "(" + regexp.QuoteMeta(souinStorageSeparator) + "|$)")
-	staleUrlRegexp := regexp.MustCompile("(^|" + regexp.QuoteMeta(souinStorageSeparator) + ")" + regexp.QuoteMeta(staleKey) + "(" + regexp.QuoteMeta(souinStorageSeparator) + "|$)")
-
 	keys := s.ParseHeaders(s.parent.getSurrogateKey(h))
 
 	for _, key := range keys {
 		_, v := s.parent.GetSurrogateControl(h)
 		if controls := s.ParseHeaders(v); len(controls) != 0 {
 			if len(controls) == 1 && controls[0] == "" {
-				s.storeTag(key, cacheKey, urlRegexp)
-				s.storeTag(stalePrefix+key, staleKey, staleUrlRegexp)
+				s.storeTag(key, cacheKey)
+				s.storeTag(stalePrefix+key, staleKey)
 
 				continue
 			}
 			for _, control := range controls {
 				if s.parent.candidateStore(control) {
-					s.storeTag(key, cacheKey, urlRegexp)
-					s.storeTag(stalePrefix+key, staleKey, staleUrlRegexp)
+					s.storeTag(key, cacheKey)
+					s.storeTag(stalePrefix+key, staleKey)
 				}
 			}
 		} else {
-			s.storeTag(key, cacheKey, urlRegexp)
-			s.storeTag(stalePrefix+key, staleKey, staleUrlRegexp)
+			s.storeTag(key, cacheKey)
+			s.storeTag(stalePrefix+key, staleKey)
 		}
 	}
 

plugins/tyk/override/cache/surrogate/providers/common.go

@@ -106,7 +106,33 @@ func (s *baseStorage) init(config configurationtypes.AbstractConfigurationInterf
 	s.mu = sync.Mutex{}
 }
 
-func (s *baseStorage) storeTag(tag string, cacheKey string, re *regexp.Regexp) {
+// containsCacheKey checks if the cacheKey already exists in the comma-separated currentValue.
+// This is much faster than regex matching, especially for long strings.
+func containsCacheKey(currentValue, cacheKey string) bool {
+	if currentValue == "" {
+		return false
+	}
+	// Check for exact match at various positions:
+	// 1. Exact match of entire string
+	if currentValue == cacheKey {
+		return true
+	}
+	// 2. At the beginning: "cacheKey,"
+	if strings.HasPrefix(currentValue, cacheKey+souinStorageSeparator) {
+		return true
+	}
+	// 3. At the end: ",cacheKey"
+	if strings.HasSuffix(currentValue, souinStorageSeparator+cacheKey) {
+		return true
+	}
+	// 4. In the middle: ",cacheKey,"
+	if strings.Contains(currentValue, souinStorageSeparator+cacheKey+souinStorageSeparator) {
+		return true
+	}
+	return false
+}
+
+func (s *baseStorage) storeTag(tag string, cacheKey string) {
 	defer s.mu.Unlock()
 	s.mu.Lock()
 	currentValue, b := s.Storage.Load(tag)
@@ -115,7 +141,7 @@ func (s *baseStorage) storeTag(tag string, cacheKey string, re *regexp.Regexp) {
 		b = s.dynamic
 	}
 	if s.dynamic || b {
-		if !re.MatchString(currentValue.(string)) {
+		if !containsCacheKey(currentValue.(string), cacheKey) {
 			s.logger.Debugf("Store the tag %s", tag)
 			s.Storage.Store(tag, currentValue.(string)+souinStorageSeparator+cacheKey)
 		}
@@ -177,28 +203,25 @@ func (s *baseStorage) Store(response *http.Response, cacheKey string) error {
 	cacheKey = url.QueryEscape(cacheKey)
 	staleKey := stalePrefix + cacheKey
 
-	urlRegexp := regexp.MustCompile("(^|" + regexp.QuoteMeta(souinStorageSeparator) + ")" + regexp.QuoteMeta(cacheKey) + "(" + regexp.QuoteMeta(souinStorageSeparator) + "|$)")
-	staleUrlRegexp := regexp.MustCompile("(^|" + regexp.QuoteMeta(souinStorageSeparator) + ")" + regexp.QuoteMeta(staleKey) + "(" + regexp.QuoteMeta(souinStorageSeparator) + "|$)")
-
 	keys := s.ParseHeaders(s.parent.getSurrogateKey(h))
 
 	for _, key := range keys {
 		if controls := s.ParseHeaders(s.parent.GetSurrogateControl(h)); len(controls) != 0 {
 			if len(controls) == 1 && controls[0] == "" {
-				s.storeTag(key, cacheKey, urlRegexp)
-				s.storeTag(stalePrefix+key, staleKey, staleUrlRegexp)
+				s.storeTag(key, cacheKey)
+				s.storeTag(stalePrefix+key, staleKey)
 
 				continue
 			}
 			for _, control := range controls {
 				if s.parent.candidateStore(control) {
-					s.storeTag(key, cacheKey, urlRegexp)
-					s.storeTag(stalePrefix+key, staleKey, staleUrlRegexp)
+					s.storeTag(key, cacheKey)
+					s.storeTag(stalePrefix+key, staleKey)
 				}
 			}
 		} else {
-			s.storeTag(key, cacheKey, urlRegexp)
-			s.storeTag(stalePrefix+key, staleKey, staleUrlRegexp)
+			s.storeTag(key, cacheKey)
+			s.storeTag(stalePrefix+key, staleKey)
 		}
 	}