Oss 248 refactor y alphabet detectors

pkg/detectors/yandex/yandex.go

@@ -2,8 +2,10 @@ package yandex
 
 import (
 	"context"
-	regexp "github.com/wasilibs/go-re2"
+	"fmt"
+	"io"
 	"net/http"
+	"regexp"
 	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
@@ -44,17 +46,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		}
 
 		if verify {
-			req, err := http.NewRequestWithContext(ctx, "GET", "https://dictionary.yandex.net/api/v1/dicservice.json/getLangs?key="+resMatch, nil)
-			if err != nil {
-				continue
-			}
-			res, err := client.Do(req)
-			if err == nil {
-				defer res.Body.Close()
-				if res.StatusCode >= 200 && res.StatusCode < 300 {
-					s1.Verified = true
-				}
-			}
+			isVerified, err := verifyMatch(ctx, client, resMatch)
+			s1.Verified = isVerified
+			s1.SetVerificationError(err, resMatch)
 		}
 
 		results = append(results, s1)
@@ -63,6 +57,34 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 	return results, nil
 }
 
+func verifyMatch(ctx context.Context, client *http.Client, apiKey string) (bool, error) {
+	// Reference: https://yandex.com/dev/dictionary/doc/dg/reference/getLangs.html
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("https://dictionary.yandex.net/api/v1/dicservice.json/getLangs?key=%s", apiKey), http.NoBody)
+	if err != nil {
+		return false, err
+	}
+
+	res, err := client.Do(req)
+	if err != nil {
+		return false, err
+	}
+	defer func() {
+		_, _ = io.Copy(io.Discard, res.Body)
+		_ = res.Body.Close()
+	}()
+
+	switch res.StatusCode {
+	case http.StatusOK:
+		return true, nil
+	case http.StatusPaymentRequired:
+		return false, fmt.Errorf("blocked: %d", res.StatusCode)
+	case http.StatusUnauthorized, http.StatusForbidden:
+		return false, nil
+	default:
+		return false, fmt.Errorf("unexpected status code: %d", res.StatusCode)
+	}
+}
+
 func (s Scanner) Type() detectorspb.DetectorType {
 	return detectorspb.DetectorType_Yandex
 }

pkg/detectors/yelp/yelp.go

@@ -3,11 +3,11 @@ package yelp
 import (
 	"context"
 	"fmt"
+	"io"
 	"net/http"
+	"regexp"
 	"strings"
 
-	regexp "github.com/wasilibs/go-re2"
-
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@@ -48,30 +48,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		}
 
 		if verify {
-
 			client := s.client
 			if client == nil {
 				client = defaultClient
 			}
 
-			req, err := http.NewRequestWithContext(ctx, "GET", "https://api.yelp.com/v3/businesses/search?term=delis&latitude=37.786882&longitude=-122.399972", nil)
-			if err != nil {
-				continue
-			}
-			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
-			res, err := client.Do(req)
-			if err == nil {
-				defer res.Body.Close()
-				if res.StatusCode >= 200 && res.StatusCode < 300 {
-					s1.Verified = true
-				} else if res.StatusCode == 401 || res.StatusCode == 403 {
-					// The secret is determinately not verified (nothing to do)
-				} else {
-					s1.SetVerificationError(fmt.Errorf("unexpected HTTP response status %d", res.StatusCode), resMatch)
-				}
-			} else {
-				s1.SetVerificationError(err, resMatch)
-			}
+			isVerified, err := verifyMatch(ctx, client, resMatch)
+			s1.Verified = isVerified
+			s1.SetVerificationError(err, resMatch)
 		}
 
 		results = append(results, s1)
@@ -80,6 +64,35 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 	return results, nil
 }
 
+func verifyMatch(ctx context.Context, client *http.Client, apiKey string) (bool, error) {
+	// Reference: https://docs.developer.yelp.com/docs/fusion-intro
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://api.yelp.com/v3/businesses/search?term=delis&latitude=37.786882&longitude=-122.399972", http.NoBody)
+	if err != nil {
+		return false, err
+	}
+
+	// Yelp requires "Bearer " prefix in Authorization header
+	req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", apiKey))
+
+	res, err := client.Do(req)
+	if err != nil {
+		return false, err
+	}
+	defer func() {
+		_, _ = io.Copy(io.Discard, res.Body)
+		_ = res.Body.Close()
+	}()
+
+	switch res.StatusCode {
+	case http.StatusOK:
+		return true, nil
+	case http.StatusUnauthorized:
+		return false, nil
+	default:
+		return false, fmt.Errorf("unexpected status code: %d", res.StatusCode)
+	}
+}
+
 func (s Scanner) Type() detectorspb.DetectorType {
 	return detectorspb.DetectorType_Yelp
 }

pkg/detectors/yelp/yelp_integration_test.go

@@ -62,7 +62,7 @@ func TestYelp_FromChunk(t *testing.T) {
 			s:    Scanner{},
 			args: args{
 				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a yelp secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				data:   []byte(fmt.Sprintf("You can find a yelp secret %s within but not valid", inactiveSecret)),
 				verify: true,
 			},
 			want: []detectors.Result{
@@ -131,15 +131,20 @@ func TestYelp_FromChunk(t *testing.T) {
 				if len(got[i].Raw) == 0 {
 					t.Fatalf("no raw secret present: \n %+v", got[i])
 				}
-				got[i].Raw = nil
 
 				if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
 					t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
 				}
 			}
-			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
-			if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
-				t.Errorf("SlackWebhook.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+			// Fixed: Added cmpopts.IgnoreUnexported to handle unexported fields
+			ignoreOpts := []cmp.Option{
+				cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError"),
+				cmpopts.IgnoreUnexported(detectors.Result{}),  // ← This was missing
+			}
+
+			if diff := cmp.Diff(got, tt.want, ignoreOpts...); diff != "" {
+				t.Errorf("Yelp.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
 			}
 		})
 	}

pkg/detectors/youneedabudget/youneedabudget.go

@@ -3,8 +3,9 @@ package youneedabudget
 import (
 	"context"
 	"fmt"
-	regexp "github.com/wasilibs/go-re2"
+	"io"
 	"net/http"
+	"regexp"
 	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
@@ -20,14 +21,15 @@ var _ detectors.Detector = (*Scanner)(nil)
 var (
 	client = common.SaneHttpClient()
 
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
-	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"youneedabudget"}) + `\b([0-9a-f]{64})\b`)
+	// YNAB Personal Access Tokens are 64-character hexadecimal strings
+	// Based on documentation: access tokens are used with Bearer authentication
+	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"youneedabudget", "ynab"}) + `([A-Za-z0-9_-]{43,44})`)
 )
 
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
 func (s Scanner) Keywords() []string {
-	return []string{"youneedabudget"}
+	return []string{"youneedabudget", "ynab"}
 }
 
 // FromData will find and optionally verify YouNeedABudget secrets in a given set of bytes.
@@ -45,18 +47,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		}
 
 		if verify {
-			req, err := http.NewRequestWithContext(ctx, "GET", "https://api.youneedabudget.com/v1/user", nil)
-			if err != nil {
-				continue
-			}
-			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
-			res, err := client.Do(req)
-			if err == nil {
-				defer res.Body.Close()
-				if res.StatusCode >= 200 && res.StatusCode < 300 {
-					s1.Verified = true
-				}
-			}
+			isVerified, err := verifyMatch(ctx, client, resMatch)
+			s1.Verified = isVerified
+			s1.SetVerificationError(err, resMatch)
 		}
 
 		results = append(results, s1)
@@ -65,6 +58,35 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 	return results, nil
 }
 
+func verifyMatch(ctx context.Context, client *http.Client, token string) (bool, error) {
+	// Reference: https://api.youneedabudget.com/#authentication-overview
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://api.youneedabudget.com/v1/user", http.NoBody)
+	if err != nil {
+		return false, err
+	}
+
+	// YNAB API requires Bearer authentication in Authorization header
+	req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))
+
+	res, err := client.Do(req)
+	if err != nil {
+		return false, err
+	}
+	defer func() {
+		_, _ = io.Copy(io.Discard, res.Body)
+		_ = res.Body.Close()
+	}()
+
+	switch res.StatusCode {
+	case http.StatusOK:
+		return true, nil
+	case http.StatusUnauthorized, http.StatusForbidden:
+		return false, nil
+	default:
+		return false, fmt.Errorf("unexpected status code: %d", res.StatusCode)
+	}
+}
+
 func (s Scanner) Type() detectorspb.DetectorType {
 	return detectorspb.DetectorType_YouNeedABudget
 }

pkg/detectors/youneedabudget/youneedabudget_integration_test.go

@@ -19,7 +19,7 @@ import (
 func TestYouNeedABudget_FromChunk(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
 	defer cancel()
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors6")
 	if err != nil {
 		t.Fatalf("could not get test secrets from GCP: %s", err)
 	}

pkg/detectors/youneedabudget/youneedabudget_test.go

@@ -12,8 +12,8 @@ import (
 )
 
 var (
-	validPattern   = "1f80bdfa73f8e9e50445de3a5a52fbe585fafe0e26d6fccf090b11153775c43d"
-	invalidPattern = "1f?0bdfa73f8e9e50445de3a5a52fbe585fafe0e26d6fccf090b11153775c43d"
+	validPattern   = "fkyzQu-fAqA0922twkqvwAl1k8H_RAzgmEl6b6e8Qfg"
+	invalidPattern = "DRntlQ5CkhA_3tnhQpobw1D0K6OEabiV80"
 	keyword        = "youneedabudget"
 )
 
@@ -30,6 +30,11 @@ func TestYouNeedABudget_Pattern(t *testing.T) {
 			input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
 			want:  []string{validPattern},
 		},
+		{
+			name:  "valid pattern - without quotes",
+			input: fmt.Sprintf("%s token = %s", keyword, validPattern),
+			want:  []string{validPattern},
+		},
 		{
 			name:  "valid pattern - ignore duplicate",
 			input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),