Skip to content

Commit bd64a83

Browse files
kashifkhan0771kashifkhan0771
authored
Added new detector for Twilio APIKey (#3803)
1 parent ccab741 commit bd64a83

File tree

6 files changed

+367
-7
lines changed

6 files changed

+367
-7
lines changed

‎pkg/detectors/twilioapikey/twilioapikey.go

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
package twilioapikey
2+
3+
import (
4+
"context"
5+
"encoding/json"
6+
"fmt"
7+
"io"
8+
"net/http"
9+
10+
regexp "github.com/wasilibs/go-re2"
11+
12+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
13+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
14+
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
15+
)
16+
17+
type Scanner struct {
18+
detectors.DefaultMultiPartCredentialProvider
19+
client *http.Client
20+
}
21+
22+
// Ensure the Scanner satisfies the interface at compile time.
23+
var _ detectors.Detector = (*Scanner)(nil)
24+
25+
var (
26+
defaultClient = common.SaneHttpClient()
27+
apiKeyPat = regexp.MustCompile(`\bSK[a-zA-Z0-9]{32}\b`)
28+
secretPat = regexp.MustCompile(`\b[0-9a-zA-Z]{32}\b`)
29+
)
30+
31+
type serviceResponse struct {
32+
Services []struct {
33+
FriendlyName string `json:"friendly_name"` // friendly name of a service
34+
SID string `json:"sid"` // object id of service
35+
AccountSID string `json:"account_sid"` // account sid
36+
} `json:"services"`
37+
}
38+
39+
func (s Scanner) getClient() *http.Client {
40+
if s.client != nil {
41+
return s.client
42+
}
43+
44+
return defaultClient
45+
}
46+
47+
// Keywords are used for efficiently pre-filtering chunks.
48+
// Use identifiers in the secret preferably, or the provider name.
49+
func (s Scanner) Keywords() []string {
50+
return []string{"twilio"}
51+
}
52+
53+
// FromData will find and optionally verify Twilio secrets in a given set of bytes.
54+
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
55+
dataStr := string(data)
56+
57+
apiKeyMatches := apiKeyPat.FindAllString(dataStr, -1)
58+
secretMatches := secretPat.FindAllString(dataStr, -1)
59+
60+
for _, apiKey := range apiKeyMatches {
61+
for _, secret := range secretMatches {
62+
s1 := detectors.Result{
63+
DetectorType: detectorspb.DetectorType_Twilio,
64+
Raw: []byte(apiKey),
65+
RawV2: []byte(apiKey + secret),
66+
Redacted: secret,
67+
ExtraData: make(map[string]string),
68+
}
69+
70+
if verify {
71+
extraData, isVerified, verificationErr := verifyTwilioAPIKey(ctx, s.getClient(), apiKey, secret)
72+
s1.Verified = isVerified
73+
s1.SetVerificationError(verificationErr)
74+
75+
for key, value := range extraData {
76+
s1.ExtraData[key] = value
77+
}
78+
79+
if s1.Verified {
80+
s1.AnalysisInfo = map[string]string{"key": apiKey, "sid": secret}
81+
}
82+
}
83+
84+
results = append(results, s1)
85+
}
86+
}
87+
88+
return results, nil
89+
}
90+
91+
func (s Scanner) Type() detectorspb.DetectorType {
92+
return detectorspb.DetectorType_TwilioApiKey
93+
}
94+
95+
func (s Scanner) Description() string {
96+
return "Twilio is a cloud communications platform that allows software developers to programmatically make and receive phone calls, send and receive text messages, and perform other communication functions using its web service APIs."
97+
}
98+
99+
func verifyTwilioAPIKey(ctx context.Context, client *http.Client, apiKey, secret string) (map[string]string, bool, error) {
100+
req, err := http.NewRequestWithContext(ctx, "GET", "https://verify.twilio.com/v2/Services", nil)
101+
if err != nil {
102+
return nil, false, nil
103+
}
104+
105+
req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
106+
req.Header.Add("Accept", "*/*")
107+
req.SetBasicAuth(apiKey, secret)
108+
109+
resp, err := client.Do(req)
110+
if err != nil {
111+
return nil, false, nil
112+
}
113+
defer func() {
114+
_, _ = io.Copy(io.Discard, resp.Body)
115+
_ = resp.Body.Close()
116+
}()
117+
118+
switch resp.StatusCode {
119+
case http.StatusOK:
120+
extraData := make(map[string]string)
121+
var serviceResponse serviceResponse
122+
123+
if err := json.NewDecoder(resp.Body).Decode(&serviceResponse); err == nil && len(serviceResponse.Services) > 0 { // no error in parsing and have at least one service
124+
service := serviceResponse.Services[0]
125+
extraData["friendly_name"] = service.FriendlyName
126+
extraData["account_sid"] = service.AccountSID
127+
}
128+
129+
return extraData, true, nil
130+
case http.StatusUnauthorized, http.StatusForbidden:
131+
return nil, false, nil
132+
default:
133+
return nil, false, fmt.Errorf("unexpected HTTP response status %d", resp.StatusCode)
134+
}
135+
}

‎pkg/detectors/twilioapikey/twilioapikey_integration_test.go

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
//go:build detectors
2+
// +build detectors
3+
4+
package twilioapikey
5+
6+
import (
7+
"context"
8+
"fmt"
9+
"testing"
10+
"time"
11+
12+
"github.com/google/go-cmp/cmp"
13+
"github.com/google/go-cmp/cmp/cmpopts"
14+
15+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
16+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
17+
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
18+
)
19+
20+
func TestTwilioAPIKey_FromChunk(t *testing.T) {
21+
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
22+
defer cancel()
23+
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
24+
if err != nil {
25+
t.Fatalf("could not get test secrets from GCP: %s", err)
26+
}
27+
28+
secret := testSecrets.MustGetField("TWILLIO_SECRET")
29+
secretInactive := testSecrets.MustGetField("TWILLIO_SECRET_INACTIVE")
30+
apiKey := testSecrets.MustGetField("TWILLIO_APIKEY")
31+
32+
type args struct {
33+
ctx context.Context
34+
data []byte
35+
verify bool
36+
}
37+
tests := []struct {
38+
name string
39+
s Scanner
40+
args args
41+
want []detectors.Result
42+
wantErr bool
43+
wantVerificationErr bool
44+
}{
45+
{
46+
name: "found, verified",
47+
s: Scanner{},
48+
args: args{
49+
ctx: context.Background(),
50+
data: []byte(fmt.Sprintf("You can find a twillio secret %s within awsId %s", secret, apiKey)),
51+
verify: true,
52+
},
53+
want: []detectors.Result{
54+
{
55+
DetectorType: detectorspb.DetectorType_Twilio,
56+
Verified: true,
57+
Redacted: secret,
58+
RawV2: []byte(apiKey + secret),
59+
ExtraData: map[string]string{
60+
"account_sid": "ACa5b6165773490f33f226d71e7ffacff5",
61+
"friendly_name": "MyServiceName",
62+
},
63+
},
64+
},
65+
wantErr: false,
66+
},
67+
{
68+
name: "found, not verified",
69+
s: Scanner{},
70+
args: args{
71+
ctx: context.Background(),
72+
data: []byte(fmt.Sprintf("You can find a twillio secret %s within awsId %s", secretInactive, apiKey)),
73+
verify: true,
74+
},
75+
want: []detectors.Result{
76+
{
77+
DetectorType: detectorspb.DetectorType_Twilio,
78+
Verified: false,
79+
Redacted: secretInactive,
80+
RawV2: []byte(apiKey + secretInactive),
81+
ExtraData: map[string]string{},
82+
},
83+
},
84+
wantErr: false,
85+
},
86+
{
87+
name: "not found",
88+
s: Scanner{},
89+
args: args{
90+
ctx: context.Background(),
91+
data: []byte("You cannot find the secret within"),
92+
verify: true,
93+
},
94+
want: nil,
95+
wantErr: false,
96+
},
97+
}
98+
for _, tt := range tests {
99+
t.Run(tt.name, func(t *testing.T) {
100+
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
101+
if (err != nil) != tt.wantErr {
102+
t.Errorf("Twilio.FromData() error = %v, wantErr %v", err, tt.wantErr)
103+
return
104+
}
105+
for i := range got {
106+
if len(got[i].Raw) == 0 {
107+
t.Fatalf("no raw secret present: \n %+v", got[i])
108+
}
109+
if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
110+
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
111+
}
112+
}
113+
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError", "AnalysisInfo")
114+
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
115+
t.Errorf("Twilio.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
116+
}
117+
})
118+
}
119+
}
120+
121+
func BenchmarkFromData(benchmark *testing.B) {
122+
ctx := context.Background()
123+
s := Scanner{}
124+
for name, data := range detectors.MustGetBenchmarkData() {
125+
benchmark.Run(name, func(b *testing.B) {
126+
b.ResetTimer()
127+
for n := 0; n < b.N; n++ {
128+
_, err := s.FromData(ctx, false, data)
129+
if err != nil {
130+
b.Fatal(err)
131+
}
132+
}
133+
})
134+
}
135+
}

‎pkg/detectors/twilioapikey/twilioapikey_test.go

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
package twilioapikey
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"testing"
7+
8+
"github.com/google/go-cmp/cmp"
9+
10+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
11+
"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
12+
)
13+
14+
var (
15+
validAPIKey = "SKbddcfb88fake8f7c4d9aefake7de1fc5"
16+
invalidAPIKey = "SK_bddcfb88fake8f7c4d9aefake7de1fc"
17+
validSecret = "k7JXtY3WBtUqthisisfakeZDqVcjZxYI"
18+
invalidSecret = "k6JXtY3WBtU$thisisfakeZDqVcjZxYI"
19+
keyword = "twilio"
20+
)
21+
22+
func TestTwilioAPIKey_Pattern(t *testing.T) {
23+
d := Scanner{}
24+
ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
25+
tests := []struct {
26+
name string
27+
input string
28+
want []string
29+
}{
30+
{
31+
name: "valid pattern - with keyword twilio",
32+
input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validAPIKey, keyword, validSecret),
33+
want: []string{validAPIKey + validSecret},
34+
},
35+
{
36+
name: "invalid pattern",
37+
input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidAPIKey, keyword, invalidSecret),
38+
want: []string{},
39+
},
40+
}
41+
42+
for _, test := range tests {
43+
t.Run(test.name, func(t *testing.T) {
44+
matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
45+
if len(matchedDetectors) == 0 {
46+
t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
47+
return
48+
}
49+
50+
results, err := d.FromData(context.Background(), false, []byte(test.input))
51+
if err != nil {
52+
t.Errorf("error = %v", err)
53+
return
54+
}
55+
56+
if len(results) != len(test.want) {
57+
if len(results) == 0 {
58+
t.Errorf("did not receive result")
59+
} else {
60+
t.Errorf("expected %d results, only received %d", len(test.want), len(results))
61+
}
62+
return
63+
}
64+
65+
actual := make(map[string]struct{}, len(results))
66+
for _, r := range results {
67+
if len(r.RawV2) > 0 {
68+
actual[string(r.RawV2)] = struct{}{}
69+
} else {
70+
actual[string(r.Raw)] = struct{}{}
71+
}
72+
}
73+
expected := make(map[string]struct{}, len(test.want))
74+
for _, v := range test.want {
75+
expected[v] = struct{}{}
76+
}
77+
78+
if diff := cmp.Diff(expected, actual); diff != "" {
79+
t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
80+
}
81+
})
82+
}
83+
}

‎pkg/engine/defaults/defaults.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -737,6 +737,7 @@ import (
737737
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/trufflehogenterprise"
738738
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/twelvedata"
739739
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/twilio"
740+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/twilioapikey"
740741
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/twist"
741742
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/twitch"
742743
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/twitchaccesstoken"
@@ -1582,6 +1583,7 @@ func buildDetectorList() []detectors.Detector {
15821583
&trufflehogenterprise.Scanner{},
15831584
&twelvedata.Scanner{},
15841585
&twilio.Scanner{},
1586+
&twilioapikey.Scanner{},
15851587
&twist.Scanner{},
15861588
&twitch.Scanner{},
15871589
&twitchaccesstoken.Scanner{},

0 commit comments

Comments
 (0)