Skip to content

Commit bfaddae

Browse files
nabeelalamamanfcp
nabeelalam
and
amanfcp
authored
Added Anypoint API OAuth2 Detector (#4312)
* added anypoint oauth2 detector * deleting secret from uniqueSecrets map if id-secret pair is verified --------- Co-authored-by: Amaan Ullah <[email protected]>
1 parent 907ac64 commit bfaddae

File tree

5 files changed

+383
-7
lines changed

5 files changed

+383
-7
lines changed

‎pkg/detectors/anypointoauth2/anypointoauth2.go

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
package anypointoauth2
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"io"
7+
"net/http"
8+
"strings"
9+
10+
regexp "github.com/wasilibs/go-re2"
11+
12+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
13+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
14+
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
15+
)
16+
17+
type Scanner struct {
18+
client *http.Client
19+
detectors.DefaultMultiPartCredentialProvider
20+
}
21+
22+
// Ensure the Scanner satisfies the interface at compile time.
23+
var _ detectors.Detector = (*Scanner)(nil)
24+
25+
var (
26+
defaultClient = common.SaneHttpClient()
27+
28+
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
29+
idPat = regexp.MustCompile(detectors.PrefixRegex([]string{"anypoint", "id"}) + `\b([0-9a-f]{32})\b`)
30+
secretPat = regexp.MustCompile(detectors.PrefixRegex([]string{"anypoint", "secret"}) + `\b([0-9a-fA-F]{32})\b`)
31+
32+
verificationUrl = "https://anypoint.mulesoft.com/accounts/oauth2/token"
33+
)
34+
35+
// Keywords are used for efficiently pre-filtering chunks.
36+
// Use identifiers in the secret preferably, or the provider name.
37+
func (s Scanner) Keywords() []string {
38+
return []string{"anypoint"}
39+
}
40+
41+
func (s Scanner) getClient() *http.Client {
42+
if s.client != nil {
43+
return s.client
44+
}
45+
return defaultClient
46+
}
47+
48+
// FromData will find and optionally verify Anypoint secrets in a given set of bytes.
49+
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
50+
dataStr := string(data)
51+
52+
var uniqueIDs, uniqueSecrets = make(map[string]struct{}), make(map[string]struct{})
53+
54+
for _, matches := range idPat.FindAllStringSubmatch(dataStr, -1) {
55+
uniqueIDs[matches[1]] = struct{}{}
56+
}
57+
58+
for _, matches := range secretPat.FindAllStringSubmatch(dataStr, -1) {
59+
uniqueSecrets[matches[1]] = struct{}{}
60+
}
61+
62+
for id := range uniqueIDs {
63+
for secret := range uniqueSecrets {
64+
if id == secret {
65+
// Avoid processing the same string for both id and secret.
66+
continue
67+
}
68+
69+
s1 := detectors.Result{
70+
DetectorType: detectorspb.DetectorType_AnypointOAuth2,
71+
Raw: []byte(secret),
72+
RawV2: []byte(fmt.Sprintf("%s:%s", id, secret)),
73+
}
74+
75+
if verify {
76+
client := s.getClient()
77+
isVerified, verificationErr := verifyMatch(ctx, client, id, secret)
78+
s1.Verified = isVerified
79+
s1.SetVerificationError(verificationErr)
80+
81+
}
82+
83+
results = append(results, s1)
84+
85+
if s1.Verified {
86+
// Anypoint client IDs and secrets are mapped one-to-one, so if a pair
87+
// is verified, we can remove that secret from the uniqueSecrets map.
88+
delete(uniqueSecrets, secret)
89+
break
90+
}
91+
}
92+
}
93+
94+
return
95+
}
96+
97+
func verifyMatch(ctx context.Context, client *http.Client, id, secret string) (bool, error) {
98+
payload := strings.NewReader(`{"grant_type":"client_credentials","client_id":"` + id + `","client_secret":"` + secret + `"}`)
99+
req, err := http.NewRequestWithContext(ctx, http.MethodPost, verificationUrl, payload)
100+
if err != nil {
101+
return false, err
102+
}
103+
req.Header.Add("Content-Type", "application/json")
104+
res, err := client.Do(req)
105+
if err != nil {
106+
return false, err
107+
}
108+
109+
defer func() {
110+
_, _ = io.Copy(io.Discard, res.Body)
111+
_ = res.Body.Close()
112+
}()
113+
114+
switch res.StatusCode {
115+
// The endpoint responds with status 200 for valid Organization credentials and 422 for Client credentials.
116+
case http.StatusOK, http.StatusUnprocessableEntity:
117+
return true, nil
118+
case http.StatusUnauthorized:
119+
return false, nil
120+
default:
121+
return false, fmt.Errorf("unexpected status code: %d", res.StatusCode)
122+
}
123+
}
124+
125+
func (s Scanner) Type() detectorspb.DetectorType {
126+
return detectorspb.DetectorType_AnypointOAuth2
127+
}
128+
129+
func (s Scanner) Description() string {
130+
return "Anypoint is a unified platform that allows organizations to build and manage APIs and integrations. Anypoint credentials can be used to access and manipulate these integrations and API data."
131+
}

‎pkg/detectors/anypointoauth2/anypointoauth2_integration_test.go

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
//go:build detectors
2+
// +build detectors
3+
4+
package anypointoauth2
5+
6+
import (
7+
"context"
8+
"fmt"
9+
"testing"
10+
"time"
11+
12+
"github.com/google/go-cmp/cmp"
13+
"github.com/google/go-cmp/cmp/cmpopts"
14+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
15+
16+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
17+
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
18+
)
19+
20+
func TestAnypointOAuth2_FromChunk(t *testing.T) {
21+
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
22+
defer cancel()
23+
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors6")
24+
if err != nil {
25+
t.Fatalf("could not get test secrets from GCP: %s", err)
26+
}
27+
clientID := testSecrets.MustGetField("ANYPOINT_CLIENT_ID")
28+
clientSecret := testSecrets.MustGetField("ANYPOINT_CLIENT_SECRET")
29+
inactiveSecret := testSecrets.MustGetField("ANYPOINT_INACTIVE_SECRET")
30+
31+
type args struct {
32+
ctx context.Context
33+
data []byte
34+
verify bool
35+
}
36+
tests := []struct {
37+
name string
38+
s Scanner
39+
args args
40+
want []detectors.Result
41+
wantErr bool
42+
}{
43+
{
44+
name: "found, verified",
45+
s: Scanner{},
46+
args: args{
47+
ctx: context.Background(),
48+
data: []byte(fmt.Sprintf("You can find an anypoint secret %s within anypoint organization id %s", clientSecret, clientID)),
49+
verify: true,
50+
},
51+
want: []detectors.Result{
52+
{
53+
DetectorType: detectorspb.DetectorType_AnypointOAuth2,
54+
Verified: true,
55+
},
56+
},
57+
wantErr: false,
58+
},
59+
{
60+
name: "found, unverified",
61+
s: Scanner{},
62+
args: args{
63+
ctx: context.Background(),
64+
data: []byte(fmt.Sprintf("You can find an anypoint secret %s within anypoint organization id %s but not valid", inactiveSecret, clientID)), // the secret would satisfy the regex but not pass validation
65+
verify: true,
66+
},
67+
want: []detectors.Result{
68+
{
69+
DetectorType: detectorspb.DetectorType_AnypointOAuth2,
70+
Verified: false,
71+
},
72+
},
73+
wantErr: false,
74+
},
75+
{
76+
name: "not found",
77+
s: Scanner{},
78+
args: args{
79+
ctx: context.Background(),
80+
data: []byte("You cannot find the secret within"),
81+
verify: true,
82+
},
83+
want: nil,
84+
wantErr: false,
85+
},
86+
}
87+
for _, tt := range tests {
88+
t.Run(tt.name, func(t *testing.T) {
89+
s := Scanner{}
90+
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
91+
if (err != nil) != tt.wantErr {
92+
t.Errorf("Anypoint.FromData() error = %v, wantErr %v", err, tt.wantErr)
93+
return
94+
}
95+
for i := range got {
96+
if len(got[i].Raw) == 0 {
97+
t.Fatalf("no raw secret present: \n %+v", got[i])
98+
}
99+
gotErr := ""
100+
if got[i].VerificationError() != nil {
101+
gotErr = got[i].VerificationError().Error()
102+
}
103+
wantErr := ""
104+
if tt.want[i].VerificationError() != nil {
105+
wantErr = tt.want[i].VerificationError().Error()
106+
}
107+
if gotErr != wantErr {
108+
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())
109+
}
110+
}
111+
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError", "primarySecret")
112+
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
113+
t.Errorf("AnypointOAuth2.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
114+
}
115+
})
116+
}
117+
}
118+
119+
func BenchmarkFromData(benchmark *testing.B) {
120+
ctx := context.Background()
121+
s := Scanner{}
122+
for name, data := range detectors.MustGetBenchmarkData() {
123+
benchmark.Run(name, func(b *testing.B) {
124+
b.ResetTimer()
125+
for n := 0; n < b.N; n++ {
126+
_, err := s.FromData(ctx, false, data)
127+
if err != nil {
128+
b.Fatal(err)
129+
}
130+
}
131+
})
132+
}
133+
}

‎pkg/detectors/anypointoauth2/anypointoauth2_test.go

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
package anypointoauth2
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"testing"
7+
8+
"github.com/google/go-cmp/cmp"
9+
10+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
11+
"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
12+
)
13+
14+
var (
15+
validPattern = "anypoint id: e3cd10a87f53b2dfa4b5fd606e7d9eca / secret: ACE9d7E606Df5B4AFD2B35f78A01DC3E"
16+
complexPattern = `
17+
# Secret Configuration File
18+
# Organization details
19+
ORG_NAME=my_organization
20+
ORG_ID=abcd1234-ef56-gh78-ij90-klmn1234opqr
21+
22+
# Database credentials
23+
DB_USERNAME=iamnotadmin
24+
DB_PASSWORD=8f3b6d3e7c9a2f5e
25+
26+
# OAuth tokens
27+
CLIENT_ID=e3cd10a87f53b2dfa4b5fd606e7d9eca
28+
CLIENT_SECRET=ACE9d7E606Df5B4AFD2B35f78A01DC3E
29+
30+
# API keys
31+
API_KEY=sk-ant-api03-nothing-just-some-random-api-key-1234fghijklmnopAA
32+
SECRET_KEY=1a2b3c4d-5e6f-7g8h-9i0j-k1l2m3n4o5p6
33+
34+
# Endpoints
35+
SERVICE_URL=https://api.example.com/v1/resource
36+
`
37+
invalidPattern = "anypoint id: k4lzc5ty98tnfu3a11y8gnv5vb1281as / secret: 8SBT9p4NXPYVS89EPtYV29SVT2SFcD8A"
38+
)
39+
40+
func TestAnypoint_Pattern(t *testing.T) {
41+
d := Scanner{}
42+
ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
43+
44+
tests := []struct {
45+
name string
46+
input string
47+
want []string
48+
}{
49+
{
50+
name: "valid pattern",
51+
input: fmt.Sprintf("anypoint credentials: %s", validPattern),
52+
want: []string{"e3cd10a87f53b2dfa4b5fd606e7d9eca:ACE9d7E606Df5B4AFD2B35f78A01DC3E"},
53+
},
54+
{
55+
name: "valid pattern - complex",
56+
input: fmt.Sprintf("anypoint credentials: %s", complexPattern),
57+
want: []string{"e3cd10a87f53b2dfa4b5fd606e7d9eca:ACE9d7E606Df5B4AFD2B35f78A01DC3E"},
58+
},
59+
{
60+
name: "invalid pattern",
61+
input: fmt.Sprintf("anypoint credentials: %s", invalidPattern),
62+
want: nil,
63+
},
64+
}
65+
66+
for _, test := range tests {
67+
t.Run(test.name, func(t *testing.T) {
68+
matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
69+
if len(matchedDetectors) == 0 {
70+
t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
71+
return
72+
}
73+
74+
results, err := d.FromData(context.Background(), false, []byte(test.input))
75+
if err != nil {
76+
t.Errorf("error = %v", err)
77+
return
78+
}
79+
80+
if len(results) != len(test.want) {
81+
if len(results) == 0 {
82+
t.Errorf("did not receive result")
83+
} else {
84+
t.Errorf("expected %d results, only received %d", len(test.want), len(results))
85+
}
86+
return
87+
}
88+
89+
actual := make(map[string]struct{}, len(results))
90+
for _, r := range results {
91+
if len(r.RawV2) > 0 {
92+
actual[string(r.RawV2)] = struct{}{}
93+
} else {
94+
actual[string(r.Raw)] = struct{}{}
95+
}
96+
}
97+
expected := make(map[string]struct{}, len(test.want))
98+
for _, v := range test.want {
99+
expected[v] = struct{}{}
100+
}
101+
102+
if diff := cmp.Diff(expected, actual); diff != "" {
103+
t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
104+
}
105+
})
106+
}
107+
}

0 commit comments

Comments
 (0)