perf: (issue #377) add caching and use re2 for alias scanning in files (#491)

Attempting to continue the work done in
#408. Closes #377.

### Changes

- use caching to optimize file globbing and regex compilation
- swap stdlib `regexp` out for
[go-re2](https://github.com/wasilibs/go-re2) for improved regex
compilation and pattern matching
- added a test to assert alias-finding logic works properly when given a
set of files with flag references in them

### Benchmarks

I compiled and executed locally using `local-find-code-refs --debug
--dryRun -t=$SOME_TOKEN -d=. -p=default -r=some-repo
--ignoreServiceErrors`

Ran this on a repo where the command execution never completed (waited
>30 minutes until I cancelled) and when using this branch's version, the
command was able to complete within 8m14s.

Author: khoaHyh

aliases/alias.go

@@ -7,20 +7,23 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
-	"regexp"
 	"strconv"
 	"strings"
 	"time"
 
 	"github.com/bmatcuk/doublestar/v4"
 	"github.com/iancoleman/strcase"
+	regexp "github.com/wasilibs/go-re2"
 
 	"github.com/launchdarkly/ld-find-code-refs/v2/internal/helpers"
 	"github.com/launchdarkly/ld-find-code-refs/v2/internal/log"
 	"github.com/launchdarkly/ld-find-code-refs/v2/internal/validation"
 	"github.com/launchdarkly/ld-find-code-refs/v2/options"
 )
 
+var globCache = make(map[string][]string)
+var regexCache = make(map[string]*regexp.Regexp)
+
 // GenerateAliases returns a map of flag keys to aliases based on config.
 func GenerateAliases(flags []string, aliases []options.Alias, dir string) (map[string][]string, error) {
 	allFileContents, err := processFileContent(aliases, dir)
@@ -88,10 +91,9 @@ func GenerateAliasesFromFilePattern(a options.Alias, flag, dir string, allFileCo
 	// Concatenate the contents of all files into a single byte array to be matched by specified patterns
 	fileContents := []byte{}
 	for _, path := range a.Paths {
-		absGlob := filepath.Join(dir, path)
-		matches, err := doublestar.FilepathGlob(absGlob)
+		matches, err := cacheFilepathGlob(dir, path)
 		if err != nil {
-			return nil, fmt.Errorf("filepattern '%s': could not process path glob '%s'", a.Name, absGlob)
+			return nil, fmt.Errorf("filepattern '%s': could not process path glob '%s'", a.Name, path)
 		}
 		for _, match := range matches {
 			if pathFileContents := allFileContents[match]; len(pathFileContents) > 0 {
@@ -101,7 +103,12 @@ func GenerateAliasesFromFilePattern(a options.Alias, flag, dir string, allFileCo
 	}
 
 	for _, p := range a.Patterns {
-		pattern := regexp.MustCompile(strings.ReplaceAll(p, "FLAG_KEY", flag))
+		patternStr := strings.ReplaceAll(p, "FLAG_KEY", flag)
+		pattern, ok := regexCache[patternStr]
+		if !ok {
+			pattern = regexp.MustCompile(patternStr)
+			regexCache[patternStr] = pattern
+		}
 		results := pattern.FindAllStringSubmatch(string(fileContents), -1)
 		for _, res := range results {
 			if len(res) > 1 {
@@ -157,13 +164,12 @@ func processFileContent(aliases []options.Alias, dir string) (FileContentsMap, e
 
 		paths := []string{}
 		for _, glob := range a.Paths {
-			absGlob := filepath.Join(dir, glob)
-			matches, err := doublestar.FilepathGlob(absGlob)
+			matches, err := cacheFilepathGlob(dir, glob)
 			if err != nil {
-				return nil, fmt.Errorf("filepattern '%s': could not process path glob '%s'", aliasId, absGlob)
+				return nil, fmt.Errorf("filepattern '%s': could not process path glob '%s'", aliasId, glob)
 			}
 			if matches == nil {
-				log.Info.Printf("filepattern '%s': no matching files found for alias path glob '%s'", aliasId, absGlob)
+				log.Info.Printf("filepattern '%s': no matching files found for alias path glob '%s'", aliasId, glob)
 			}
 			paths = append(paths, matches...)
 		}
@@ -188,3 +194,15 @@ func processFileContent(aliases []options.Alias, dir string) (FileContentsMap, e
 	}
 	return allFileContents, nil
 }
+
+func cacheFilepathGlob(dir, glob string) ([]string, error) {
+	absGlob := filepath.Join(dir, glob)
+	if cachedMatches, ok := globCache[absGlob]; ok {
+		return cachedMatches, nil
+	}
+
+	matches, err := doublestar.FilepathGlob(absGlob)
+	globCache[absGlob] = matches
+
+	return matches, err
+}

aliases/alias_test.go

@@ -93,17 +93,17 @@ func Test_GenerateAliases(t *testing.T) {
 			name:  "file exact pattern",
 			flags: slice(testFlagKey),
 			aliases: []o.Alias{
-				fileExactPattern(testFlagKey),
+				fileExactPattern(),
 			},
 			want: map[string][]string{testFlagKey: slice("SOME_FLAG")},
 		},
 		{
 			name:  "file wildcard pattern",
 			flags: slice(testFlagKey, testWildFlagKey),
 			aliases: []o.Alias{
-				fileWildPattern(testFlagKey),
+				fileWildPattern(),
 			},
-			want: map[string][]string{testWildFlagKey: slice("WILD_FLAG", "WILD_FLAG_SECOND_ALIAS"), testFlagKey: slice("SOME_FLAG")},
+			want: map[string][]string{testWildFlagKey: slice("WILD_FLAG", "WILD_FLAG_SECOND_ALIAS", "ABSOLUTELY_WILD"), testFlagKey: slice("SOME_FLAG")},
 		},
 		{
 			name:  "command",
@@ -198,6 +198,24 @@ func Test_processFileContent(t *testing.T) {
 	}
 }
 
+func Test_GenerateAliasesFromFilePattern(t *testing.T) {
+	expectedAliases := []string{"WILD_FLAG", "WILD_FLAG_SECOND_ALIAS", "ABSOLUTELY_WILD"}
+
+	fileContents := map[string][]byte{
+		"testdata/alias_test.txt":                                  []byte("SOME_FLAG = 'someFlag'"),
+		"testdata/wild/alias_test.txt":                             []byte("WILD_FLAG = 'wildFlag'"),
+		"testdata/wild/nested-wild/alias_test.txt":                 []byte("WILD_FLAG_SECOND_ALIAS = 'wildFlag'"),
+		"testdata/wild/nested-wild/another/another/alias_test.txt": []byte("ABSOLUTELY_WILD = 'wildFlag'"),
+	}
+
+	alias := fileWildPattern()
+
+	aliases, err := GenerateAliasesFromFilePattern(alias, testWildFlagKey, "", fileContents)
+	require.NoError(t, err)
+
+	assert.ElementsMatch(t, expectedAliases, aliases)
+}
+
 func slice(args ...string) []string {
 	return args
 }
@@ -222,15 +240,15 @@ func literal(flags []string) o.Alias {
 	return a
 }
 
-func fileExactPattern(flag string) o.Alias {
+func fileExactPattern() o.Alias {
 	a := alias(o.FilePattern)
 	pattern := "(\\w+)\\s= 'FLAG_KEY'"
 	a.Paths = []string{"testdata/alias_test.txt"}
 	a.Patterns = []string{pattern}
 	return a
 }
 
-func fileWildPattern(flag string) o.Alias {
+func fileWildPattern() o.Alias {
 	a := alias(o.FilePattern)
 	pattern := "(\\w+)\\s= 'FLAG_KEY'"
 	a.Paths = []string{"testdata/**/*.txt"}

aliases/testdata/wild/nested-wild/another/another/alias_test.txt

@@ -0,0 +1 @@
+ABSOLUTELY_WILD = 'wildFlag'

go.mod

@@ -20,7 +20,10 @@ require (
 	golang.org/x/tools v0.32.0
 )
 
-require github.com/launchdarkly/api-client-go/v15 v15.1.0
+require (
+	github.com/launchdarkly/api-client-go/v15 v15.1.0
+	github.com/wasilibs/go-re2 v1.10.0
+)
 
 require (
 	dario.cat/mergo v1.0.0 // indirect
@@ -50,6 +53,8 @@ require (
 	github.com/spf13/afero v1.12.0 // indirect
 	github.com/spf13/cast v1.7.1 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
+	github.com/tetratelabs/wazero v1.9.0 // indirect
+	github.com/wasilibs/wazero-helpers v0.0.0-20240620070341-3dff1577cd52 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/multierr v1.9.0 // indirect

go.sum

@@ -226,6 +226,12 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf
 github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
 github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
+github.com/tetratelabs/wazero v1.9.0 h1:IcZ56OuxrtaEz8UYNRHBrUa9bYeX9oVY93KspZZBf/I=
+github.com/tetratelabs/wazero v1.9.0/go.mod h1:TSbcXCfFP0L2FGkRPxHphadXPjo1T6W+CseNNY7EkjM=
+github.com/wasilibs/go-re2 v1.10.0 h1:vQZEBYZOCA9jdBMmrO4+CvqyCj0x4OomXTJ4a5/urQ0=
+github.com/wasilibs/go-re2 v1.10.0/go.mod h1:k+5XqO2bCJS+QpGOnqugyfwC04nw0jaglmjrrkG8U6o=
+github.com/wasilibs/wazero-helpers v0.0.0-20240620070341-3dff1577cd52 h1:OvLBa8SqJnZ6P+mjlzc2K7PM22rRUPE1x32G9DTPrC4=
+github.com/wasilibs/wazero-helpers v0.0.0-20240620070341-3dff1577cd52/go.mod h1:jMeV4Vpbi8osrE/pKUxRZkVaA0EX7NZN0A9/oRzgpgY=
 github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
 github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=