Optimized twitch detectors (#3987)

* avoid detecting same string for two patterns

* optimized twitch detectors

* updated test cases

Author: kashifkhan0771

pkg/detectors/twitch/twitch.go

@@ -29,8 +29,8 @@ var (
 	defaultClient = common.SaneHttpClient()
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives
-	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"twitch"}) + `\b([0-9a-z]{30})\b`)
-	idPat  = regexp.MustCompile(detectors.PrefixRegex([]string{"twitch"}) + `\b([0-9a-z]{30})\b`)
+	idPat     = regexp.MustCompile(detectors.PrefixRegex([]string{"twitch"}) + `\b([0-9a-z]{30})\b`)
+	secretPat = regexp.MustCompile(detectors.PrefixRegex([]string{"twitch"}) + `\b([0-9a-z]{30})\b`)
 )
 
 // Keywords are used for efficiently pre-filtering chunks.
@@ -43,25 +43,33 @@ func (s Scanner) Keywords() []string {
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
 	dataStr := string(data)
 
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
-	idMatches := idPat.FindAllStringSubmatch(dataStr, -1)
+	var uniqueIDMatches, uniqueSecretMatches = make(map[string]struct{}), make(map[string]struct{})
 
-	for _, match := range matches {
-		resMatch := strings.TrimSpace(match[1])
+	for _, match := range idPat.FindAllStringSubmatch(dataStr, -1) {
+		uniqueIDMatches[match[1]] = struct{}{}
+	}
+
+	for _, match := range secretPat.FindAllStringSubmatch(dataStr, -1) {
+		uniqueSecretMatches[match[1]] = struct{}{}
+	}
 
-		for _, idMatch := range idMatches {
-			resIdMatch := strings.TrimSpace(idMatch[1])
+	for id := range uniqueIDMatches {
+		for secret := range uniqueSecretMatches {
+			// as both patterns are same, to avoid same strings
+			if id == secret {
+				continue
+			}
 
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_Twitch,
-				Raw:          []byte(resMatch),
+				Raw:          []byte(id),
 			}
 
 			if verify {
 				client := s.getClient()
-				isVerified, verificationErr := verifyTwitch(ctx, client, resMatch, resIdMatch)
+				isVerified, verificationErr := verifyTwitch(ctx, client, secret, id)
 				s1.Verified = isVerified
-				s1.SetVerificationError(verificationErr, resMatch)
+				s1.SetVerificationError(verificationErr, id)
 			}
 
 			results = append(results, s1)

pkg/detectors/twitch/twitch_integration_test.go

@@ -61,14 +61,6 @@ func TestTwitch_FromChunk(t *testing.T) {
 					DetectorType: detectorspb.DetectorType_Twitch,
 					Verified:     true,
 				},
-				{
-					DetectorType: detectorspb.DetectorType_Twitch,
-					Verified:     false,
-				},
-				{
-					DetectorType: detectorspb.DetectorType_Twitch,
-					Verified:     false,
-				},
 			},
 			wantErr: false,
 		},
@@ -89,14 +81,6 @@ func TestTwitch_FromChunk(t *testing.T) {
 					DetectorType: detectorspb.DetectorType_Twitch,
 					Verified:     false,
 				},
-				{
-					DetectorType: detectorspb.DetectorType_Twitch,
-					Verified:     false,
-				},
-				{
-					DetectorType: detectorspb.DetectorType_Twitch,
-					Verified:     false,
-				},
 			},
 			wantErr:             false,
 			wantVerificationErr: true,
@@ -118,14 +102,6 @@ func TestTwitch_FromChunk(t *testing.T) {
 					DetectorType: detectorspb.DetectorType_Twitch,
 					Verified:     false,
 				},
-				{
-					DetectorType: detectorspb.DetectorType_Twitch,
-					Verified:     false,
-				},
-				{
-					DetectorType: detectorspb.DetectorType_Twitch,
-					Verified:     false,
-				},
 			},
 			wantErr:             false,
 			wantVerificationErr: true,
@@ -147,14 +123,6 @@ func TestTwitch_FromChunk(t *testing.T) {
 					DetectorType: detectorspb.DetectorType_Twitch,
 					Verified:     false,
 				},
-				{
-					DetectorType: detectorspb.DetectorType_Twitch,
-					Verified:     false,
-				},
-				{
-					DetectorType: detectorspb.DetectorType_Twitch,
-					Verified:     false,
-				},
 			},
 			wantErr: false,
 		},

pkg/detectors/twitch/twitch_test.go

@@ -30,7 +30,7 @@ func TestTwitch_Pattern(t *testing.T) {
 		{
 			name:  "valid pattern - with keyword twitch",
 			input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validId),
-			want:  []string{validKey, validId, validId, validKey},
+			want:  []string{validKey, validId},
 		},
 		{
 			name:  "invalid pattern",

pkg/detectors/twitchaccesstoken/twitchaccesstoken.go

@@ -23,7 +23,7 @@ var _ detectors.Detector = (*Scanner)(nil)
 var (
 	defaultClient = common.SaneHttpClient()
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
-	keyPat = regexp.MustCompile(`\b([0-9a-z]{30})\b`)
+	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"twitch"}) + `\b([0-9a-z]{30})\b`)
 )
 
 // Keywords are used for efficiently pre-filtering chunks.

pkg/detectors/twitchaccesstoken/twitchaccesstoken_test.go

@@ -28,9 +28,9 @@ func TestTwitchaccesstoken_Pattern(t *testing.T) {
 			want:  []string{"abc123def456ghi789jkl012mno345"},
 		},
 		{
-			name:  "get request pattern",
+			name:  "get request pattern - keyword out of range",
 			input: "curl -X GET 'https://id.twitch.tv/oauth2/validate' -H 'Authorization: OAuth xbc123def456ghi789jkl012mno345'",
-			want:  []string{"xbc123def456ghi789jkl012mno345"},
+			want:  []string{},
 		},
 		{
 			name:  "finds all matches",