Merged PR 2668744: Regex backreference support

The regex backreference support works by parsing regexes at initialization to detect the presence of backreferences such as `\1`, `\2`, etc. These are then replaced with the content of the group that they were referencing. So for example `(abc+) \1` becomes `(abc+) (abc+)`. The `tryMatch` function then does the actual comparison to ensure that the two groups actually had the same content.

Only groups that are surrounded by `\b` word boundaries are supported, such as `\b(abc+)\b`. This is because otherwise it becomes really complicated with situations like `11=111111`.

The CRS also contains this construct that makes use of a negative lookahead to assert that a backreference is _not equal_ to a group (simplified): `\b(\w+)\b != (?!\b(\1)\b)(\w+)`. Support for these special construct is therefore also implemented.

The changes to CRS rule 942130 is currently pending checkin: RE2 compatibility for 920120
SpiderLabs/owasp-modsecurity-crs#1663

Because this gets us to 100% of the CRS tests passing, I have removed code that was preventing running the CRS tests by default in the CI pipeline. So the CRS tests now always run by default.

PR URL: https://msazure.visualstudio.com/DefaultCollection/One/_git/Networking-Azwaf/pullrequest/2668744

Related work items: #5697858

Author: allanrbo

hyperscan/goenginechooser.go

@@ -60,6 +60,13 @@ func (g *goRegexpFacade) FindSubmatchIndex(b []byte) []int {
 	return g.goregexpBin.FindSubmatchIndex(b)
 }
 
+func (g *goRegexpFacade) FindAllSubmatchIndex(b []byte, n int) [][]int {
+	if g.goregexp != nil {
+		return g.goregexp.FindAllSubmatchIndex(b, n)
+	}
+	return g.goregexpBin.FindAllSubmatchIndex(b, n)
+}
+
 var hexEscapeRegexp = regexp.MustCompile(`((^|[^\\])(\\\\)*)\\x([0-9a-fA-F]{2})`)
 
 func containsHexEscapedBytes(s string) bool {

hyperscan/multiregexengine.go

@@ -20,6 +20,9 @@ type engineImpl struct {
 
 	// Special case for when searching for an empty string, because Hyperscan returns an error if it's given an empty string
 	emptyStringPatternIDs []int
+
+	// Special case for when searching for a regex with a backref, because this is not otherwise directly supported by Hyperscan and Go Regexp.
+	regexesWithBackref map[int]regexWithBackref
 }
 
 type scratchSpaceImpl struct {
@@ -38,6 +41,7 @@ func NewMultiRegexEngineFactory(dbCache DbCache) waf.MultiRegexEngineFactory {
 // NewMultiRegexEngine creates a MultiRegexEngine that uses Hyperscan in prefilter mode for the initial scan, and then uses Go regexp to re-validate matches and extract strings.
 func (f *engineFactoryImpl) NewMultiRegexEngine(mm []waf.MultiRegexEnginePattern) (engine waf.MultiRegexEngine, err error) {
 	h := &engineImpl{}
+	h.regexesWithBackref = make(map[int]regexWithBackref)
 
 	patterns := []*hs.Pattern{}
 	for _, m := range mm {
@@ -47,7 +51,21 @@ func (f *engineFactoryImpl) NewMultiRegexEngine(mm []waf.MultiRegexEnginePattern
 			continue
 		}
 
-		p := hs.NewPattern(m.Expr, 0)
+		expr := m.Expr
+
+		// Special case for when searching for a regex with a backref, because this is not otherwise directly supported by Hyperscan and Go Regexp.
+		var hasBackref bool
+		var r regexWithBackref
+		hasBackref, r, err = newRegexWithBackref(m.Expr)
+		if err != nil {
+			return
+		}
+		if hasBackref {
+			h.regexesWithBackref[m.ID] = r
+			expr = r.newRegex
+		}
+
+		p := hs.NewPattern(expr, 0)
 		p.Id = m.ID
 
 		// SingleMatch makes Hyperscan only return one match per regex. So if a regex is found multiple time, still only one match is recorded.
@@ -88,6 +106,11 @@ func (f *engineFactoryImpl) NewMultiRegexEngine(mm []waf.MultiRegexEnginePattern
 		// Make the PCRE regex compatible with Go regexp
 		expr := removePcrePossessiveQuantifier(m.Expr)
 
+		// Special case for when searching for a regex with a backref, because this is not otherwise directly supported by Hyperscan and Go Regexp.
+		if r, ok := h.regexesWithBackref[m.ID]; ok {
+			expr = r.newRegex
+		}
+
 		var r *goRegexpFacade
 		r, err = compileRegexpFacade(expr)
 		if err != nil {
@@ -140,12 +163,27 @@ func (h *engineImpl) Scan(input []byte, s waf.MultiRegexEngineScratchSpace) (mat
 
 	// Re-validate the potential matches using Go regexp
 	for _, pmID := range potentialMatches {
+		// Special case for when searching for a regex with a backref, because this is not otherwise directly supported by Hyperscan and Go Regexp.
+		if rwbh, ok := h.regexesWithBackref[pmID]; ok {
+			data, captureGroups := rwbh.tryMatch(input, h.goregexes[pmID])
+			if data != nil {
+				m := waf.MultiRegexEngineMatch{
+					ID:            pmID,
+					Data:          data,
+					CaptureGroups: captureGroups,
+				}
+				matches = append(matches, m)
+			}
+
+			continue
+		}
+
 		loc := h.goregexes[pmID].FindSubmatchIndex(input)
 		if loc == nil {
 			continue
 		}
 
-		// FindSubmatchIndex will always return an even number, because it returns pairs of start-end-locations.
+		// The length of the array returned by FindSubmatchIndex will always be an even number, because it returns pairs of start-end-locations.
 		var captureGroups [][]byte
 		for i := 0; i < len(loc); i = i + 2 {
 			if loc[i] != -1 {
@@ -161,7 +199,6 @@ func (h *engineImpl) Scan(input []byte, s waf.MultiRegexEngineScratchSpace) (mat
 			Data:          input[loc[0]:loc[1]],
 			CaptureGroups: captureGroups,
 		}
-
 		matches = append(matches, m)
 	}
 

hyperscan/multiregexengine_crsfiles_integration_test.go

@@ -67,12 +67,6 @@ func TestAllCrsReqRulesIndividually(t *testing.T) {
 				continue
 			}
 
-			// Skip this rule until we add support for backreferences
-			// TODO add support for backreferences
-			if rule.ID == 942130 {
-				continue
-			}
-
 			for itemIdx, item := range rule.Items {
 				if item.Predicate.Op != ast.Rx {
 					continue