Merge branch 'main' into gha-fix

Author: kellydunn

go.mod

@@ -41,7 +41,7 @@ require (
 	github.com/couchbase/gocb/v2 v2.10.1
 	github.com/crewjam/rfc5424 v0.1.0
 	github.com/csnewman/dextk v0.3.0
-	github.com/docker/docker v28.2.2+incompatible
+	github.com/docker/docker v28.3.3+incompatible
 	github.com/dustin/go-humanize v1.0.1
 	github.com/elastic/go-elasticsearch/v8 v8.17.1
 	github.com/envoyproxy/protoc-gen-validate v1.2.1

go.sum

@@ -483,6 +483,8 @@ github.com/docker/docker v28.2.2+incompatible h1:CjwRSksz8Yo4+RmQ339Dp/D2tGO5Jxw
 github.com/docker/docker v28.2.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/docker v28.3.2+incompatible h1:wn66NJ6pWB1vBZIilP8G3qQPqHy5XymfYn5vsqeA5oA=
 github.com/docker/docker v28.3.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI=
+github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/docker-credential-helpers v0.8.2 h1:bX3YxiGzFP5sOXWc3bTPEXdEaZSeVMrFgOr3T+zrFAo=
 github.com/docker/docker-credential-helpers v0.8.2/go.mod h1:P3ci7E3lwkZg6XiHdRKft1KckHiO9a2rNtyFbZ/ry9M=
 github.com/docker/docker-credential-helpers v0.9.3 h1:gAm/VtF9wgqJMoxzT3Gj5p4AqIjCBS4wrsOh9yRqcz8=

main.go

@@ -136,6 +136,7 @@ var (
 	gitlabScanEndpoint     = gitlabScan.Flag("endpoint", "GitLab endpoint.").Default("https://gitlab.com").String()
 	gitlabScanRepos        = gitlabScan.Flag("repo", "GitLab repo url. You can repeat this flag. Leave empty to scan all repos accessible with provided credential. Example: https://gitlab.com/org/repo.git").Strings()
 	gitlabScanToken        = gitlabScan.Flag("token", "GitLab token. Can be provided with environment variable GITLAB_TOKEN.").Envar("GITLAB_TOKEN").Required().String()
+	gitlabScanGroupIds     = gitlabScan.Flag("group-id", "GitLab group ID. If provided, it will scan the group and its subgroups. You can repeat this flag.").Strings()
 	gitlabScanIncludePaths = gitlabScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
 	gitlabScanExcludePaths = gitlabScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
 	gitlabScanIncludeRepos = gitlabScan.Flag("include-repos", `Repositories to include in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Gitlab repo full name. Example: "trufflesecurity/trufflehog", "trufflesecurity/t*"`).Strings()
@@ -788,10 +789,15 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
 			return scanMetrics, fmt.Errorf("could not create filter: %v", err)
 		}
 
+		if len(*gitlabScanRepos) > 0 && len(*gitlabScanGroupIds) > 0 {
+			return scanMetrics, fmt.Errorf("invalid config: you cannot specify both repositories and groups at the same time")
+		}
+
 		cfg := sources.GitlabConfig{
 			Endpoint:     *gitlabScanEndpoint,
 			Token:        *gitlabScanToken,
 			Repos:        *gitlabScanRepos,
+			GroupIds:     *gitlabScanGroupIds,
 			IncludeRepos: *gitlabScanIncludeRepos,
 			ExcludeRepos: *gitlabScanExcludeRepos,
 			Filter:       filter,

pkg/detectors/aha/aha.go

@@ -34,6 +34,14 @@ func (s Scanner) Keywords() []string {
 	return []string{"aha.io"}
 }
 
+func (s Scanner) Type() detectorspb.DetectorType {
+	return detectorspb.DetectorType_Aha
+}
+
+func (s Scanner) Description() string {
+	return "Aha is a product management software suite. Aha API keys can be used to access and modify product data and workflows."
+}
+
 func (s Scanner) getClient() *http.Client {
 	if s.client != nil {
 		return s.client
@@ -44,30 +52,39 @@ func (s Scanner) getClient() *http.Client {
 // FromData will find and optionally verify Aha secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
 	dataStr := string(data)
+
+	var uniqueFoundUrls = make(map[string]struct{})
+
 	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
-	URLmatches := URLPat.FindAllStringSubmatch(dataStr, -1)
 
-	resURLMatch := "aha.io"
-	for _, URLmatch := range URLmatches {
-		resURLMatch = strings.TrimSpace(URLmatch[1])
+	for _, match := range URLPat.FindAllStringSubmatch(dataStr, -1) {
+		uniqueFoundUrls[match[1]] = struct{}{}
 	}
 
-	for _, match := range matches {
-		resMatch := strings.TrimSpace(match[1])
-
-		s1 := detectors.Result{
-			DetectorType: detectorspb.DetectorType_Aha,
-			Raw:          []byte(resMatch),
-		}
+	// if no url was found use the default
+	if len(uniqueFoundUrls) == 0 {
+		uniqueFoundUrls["aha.io"] = struct{}{}
+	}
 
-		if verify {
-			client := s.getClient()
-			isVerified, verificationErr := verifyAha(ctx, client, resMatch, resURLMatch)
-			s1.Verified = isVerified
-			s1.SetVerificationError(verificationErr, resMatch)
+	for _, match := range matches {
+		for url := range uniqueFoundUrls {
+			resMatch := strings.TrimSpace(match[1])
+
+			s1 := detectors.Result{
+				DetectorType: detectorspb.DetectorType_Aha,
+				Raw:          []byte(resMatch),
+				RawV2:        []byte(resMatch + url),
+			}
+
+			if verify {
+				client := s.getClient()
+				isVerified, verificationErr := verifyAha(ctx, client, resMatch, url)
+				s1.Verified = isVerified
+				s1.SetVerificationError(verificationErr, resMatch)
+			}
+
+			results = append(results, s1)
 		}
-
-		results = append(results, s1)
 	}
 
 	return results, nil
@@ -98,11 +115,3 @@ func verifyAha(ctx context.Context, client *http.Client, resMatch, resURLMatch s
 		return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
 	}
 }
-
-func (s Scanner) Type() detectorspb.DetectorType {
-	return detectorspb.DetectorType_Aha
-}
-
-func (s Scanner) Description() string {
-	return "Aha is a product management software suite. Aha API keys can be used to access and modify product data and workflows."
-}

pkg/detectors/aha/aha_test.go

@@ -2,8 +2,6 @@ package aha
 
 import (
 	"context"
-	"fmt"
-	"strings"
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
@@ -12,52 +10,62 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
 )
 
-var (
-	validPattern   = "00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff/example.aha.io"
-	invalidPattern = "00112233445566778899aabbCC$%eeff00112233445566778899aabbccddeeff/example.fake.io"
-)
-
 func TestAha_Pattern(t *testing.T) {
 	d := Scanner{}
 	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
 
-	key := strings.Split(validPattern, "/")[0]
-	url := strings.Split(validPattern, "/")[1]
-
 	tests := []struct {
 		name  string
 		input string
 		want  []string
 	}{
 		{
-			name:  "valid pattern",
-			input: fmt.Sprintf("aha.io = '%s'", validPattern),
-			want:  []string{key},
-		},
-		{
-			name:  "valid pattern - detect URL far away from keyword",
-			input: fmt.Sprintf("aha.io = '%s\n URL is not close to the keyword but should be detected %s'", key, url),
-			want:  []string{key},
+			name: "valid pattern",
+			input: `
+				[INFO] sending request to the aha.io API
+				[DEBUG] using key = 81a1411a7e276fd88819df3137eb406e0f281f8a8c417947ca4b025890c8541c
+				[DEBUG] using host = example.aha.io
+				[INFO] response received: 200 OK
+			`,
+			want: []string{"81a1411a7e276fd88819df3137eb406e0f281f8a8c417947ca4b025890c8541cexample.aha.io"},
 		},
 		{
-			name:  "valid pattern - key out of prefix range",
-			input: fmt.Sprintf("aha.io keyword is not close to the real key and secret = '%s'", validPattern),
-			want:  nil,
+			name: "valid pattern - key out of prefix range",
+			input: `
+				[INFO] sending request to the aha.io API
+				[WARN] Do not commit the secrets
+				[DEBUG] using key = 81a1411a7e276fd88819df3137eb406e0f281f8a8c417947ca4b025890c8541c
+				[DEBUG] using host = example.aha.io
+				[INFO] response received: 200 OK
+			`,
+			want: nil,
 		},
 		{
-			name:  "valid pattern - only key",
-			input: fmt.Sprintf("aha.io %s", key),
-			want:  []string{key},
+			name: "valid pattern - only key",
+			input: `
+				[INFO] sending request to the aha.io API
+				[DEBUG] using key = 81a1411a7e276fd88819df3137eb406e0f281f8a8c417947ca4b025890c8541c
+				[INFO] response received: 200 OK
+			`,
+			want: []string{"81a1411a7e276fd88819df3137eb406e0f281f8a8c417947ca4b025890c8541caha.io"},
 		},
 		{
-			name:  "valid pattern - only URL",
-			input: fmt.Sprintf("aha.io %s", url),
-			want:  nil,
+			name: "valid pattern - only URL",
+			input: `
+				[INFO] sending request to the example.aha.io API
+				[INFO] response received: 200 OK
+			`,
+			want: nil,
 		},
 		{
-			name:  "invalid pattern",
-			input: fmt.Sprintf("aha.io %s", invalidPattern),
-			want:  nil,
+			name: "invalid pattern",
+			input: `
+				[INFO] sending request to the aha.io API
+				[DEBUG] using key = 81a1411a7e276fd88819df3137eJ406e0f281f8a8c417947ca4b025890c8541c
+				[DEBUG] using host = 1test.aha.io
+				[INFO] response received: 200 OK
+			`,
+			want: nil,
 		},
 	}
 

pkg/detectors/github/v1/github_old.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
+	"strings"
 
 	regexp "github.com/wasilibs/go-re2"
 
@@ -27,7 +28,8 @@ func (Scanner) CloudEndpoint() string { return "https://api.github.com" }
 var (
 	// Oauth token
 	// https://developer.github.com/v3/#oauth2-token-sent-in-a-header
-	keyPat = regexp.MustCompile(`(?i)(?:github|gh|pat|token)[^\.].{0,40}[ =:'"]+([a-f0-9]{40})\b`)
+	// the middle regex `\b[a-zA-Z0-9.\/?=&]{0,40}` is to match the prefix of token match to avoid processing common known patterns
+	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"github", "gh", "pat", "token"}) + `\b[a-zA-Z0-9.\/?=&]{0,40}` + `\b([a-f0-9]{40})\b`)
 
 	// TODO: Oauth2 client_id and client_secret
 	// https://developer.github.com/v3/#oauth2-keysecret
@@ -55,13 +57,25 @@ type HeaderInfo struct {
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
 func (s Scanner) Keywords() []string {
-	return []string{"github", "gh", "pat", "token"}
+	return []string{"github", "gh"}
+}
+
+func (s Scanner) Type() detectorspb.DetectorType {
+	return detectorspb.DetectorType_Github
+}
+
+func (s Scanner) Description() string {
+	return "GitHub is a web-based platform used for version control and collaborative software development. GitHub tokens can be used to access and modify repositories and other resources."
 }
 
 var ghFalsePositives = map[detectors.FalsePositive]struct{}{
 	detectors.FalsePositive("github commit"): {},
 }
 
+var ghKnownNonSensitivePrefixes = []string{
+	"avatars.githubusercontent.com", // github avatar urls prefix
+}
+
 // FromData will find and optionally verify GitHub secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
 	dataStr := string(data)
@@ -70,7 +84,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 
 	for _, match := range matches {
 		// First match is entire regex, second is the first group.
-
+		matchPrefix := match[0]
 		token := match[1]
 
 		// Note that this false positive check happens **before** verification! I don't know why it's written this way
@@ -79,6 +93,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			continue
 		}
 
+		// to avoid false positives
+		if isKnownNonSensitiveCommonPrefix(matchPrefix) {
+			continue
+		}
+
+		if detectors.StringShannonEntropy(token) < 3.5 {
+			continue
+		}
+
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Github,
 			Raw:          []byte(token),
@@ -180,10 +203,16 @@ func SetHeaderInfo(headers *HeaderInfo, s1 *detectors.Result) {
 	}
 }
 
-func (s Scanner) Type() detectorspb.DetectorType {
-	return detectorspb.DetectorType_Github
-}
+// isKnownNonSensitiveCommonPrefix checks if the given prefix is a known, non-sensitive value.
+// The GitHub v1 detector uses a broad regex that can capture many false positives.
+// This function helps filter out matches that begin with common, safe prefixes.
+// Example: avatars.githubusercontent.com/u/56769451?u=088102b6160822bc68c25a2a5df170080d0b16a2
+func isKnownNonSensitiveCommonPrefix(matchPrefix string) bool {
+	for _, prefix := range ghKnownNonSensitivePrefixes {
+		if strings.Contains(matchPrefix, prefix) {
+			return true
+		}
+	}
 
-func (s Scanner) Description() string {
-	return "GitHub is a web-based platform used for version control and collaborative software development. GitHub tokens can be used to access and modify repositories and other resources."
+	return false
 }