Skip to content

Commit 2a01091

Browse files
0x10x1
authored
Add Flexport detector (#3633)
* updating protos * add flexport detector and test * adding to defaults * remove extradata from verification * update pb
1 parent 7e60ca3 commit 2a01091

File tree

5 files changed

+357
-6
lines changed

5 files changed

+357
-6
lines changed

‎pkg/detectors/flexport/flexport.go

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
package flexport
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"io"
7+
"net/http"
8+
9+
regexp "github.com/wasilibs/go-re2"
10+
11+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
12+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
13+
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
14+
)
15+
16+
type Scanner struct {
17+
client *http.Client
18+
}
19+
20+
// Ensure the Scanner satisfies the interface at compile time.
21+
var _ detectors.Detector = (*Scanner)(nil)
22+
23+
var (
24+
defaultClient = common.SaneHttpClient()
25+
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
26+
keyPat = regexp.MustCompile(`\b(shltm_[0-9a-zA-Z-_]{40})`)
27+
)
28+
29+
// Keywords are used for efficiently pre-filtering chunks.
30+
// Use identifiers in the secret preferably, or the provider name.
31+
func (s Scanner) Keywords() []string {
32+
return []string{"shltm_"}
33+
}
34+
35+
// FromData will find and optionally verify Flexport secrets in a given set of bytes.
36+
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
37+
dataStr := string(data)
38+
39+
uniqueMatches := make(map[string]struct{})
40+
for _, match := range keyPat.FindAllStringSubmatch(dataStr, -1) {
41+
uniqueMatches[match[1]] = struct{}{}
42+
}
43+
44+
for match := range uniqueMatches {
45+
s1 := detectors.Result{
46+
DetectorType: detectorspb.DetectorType_Flexport,
47+
Raw: []byte(match),
48+
ExtraData: map[string]string{
49+
"rotation_guide": "https://howtorotate.com/docs/tutorials/flexport/",
50+
},
51+
}
52+
53+
if verify {
54+
client := s.client
55+
if client == nil {
56+
client = defaultClient
57+
}
58+
59+
isVerified, verificationErr := verifyMatch(ctx, client, match)
60+
s1.Verified = isVerified
61+
s1.SetVerificationError(verificationErr, match)
62+
}
63+
64+
results = append(results, s1)
65+
}
66+
67+
return
68+
}
69+
70+
func verifyMatch(ctx context.Context, client *http.Client, token string) (bool, error) {
71+
// docs: https://docs.logistics-api.flexport.com/2024-04/tag/Webhooks#operation/GetWebhook
72+
url := "https://logistics-api.flexport.com/logistics/api/2024-04/webhooks"
73+
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
74+
if err != nil {
75+
return false, nil
76+
}
77+
req.Header.Set("Authorization", "Bearer "+token)
78+
79+
res, err := client.Do(req)
80+
if err != nil {
81+
return false, err
82+
}
83+
defer func() {
84+
_, _ = io.Copy(io.Discard, res.Body)
85+
_ = res.Body.Close()
86+
}()
87+
88+
switch res.StatusCode {
89+
case http.StatusOK, http.StatusForbidden:
90+
// If the endpoint returns useful information, we can return it as a map.
91+
return true, nil
92+
case http.StatusUnauthorized:
93+
// The secret is determinately not verified (nothing to do)
94+
return false, nil
95+
default:
96+
return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
97+
}
98+
}
99+
100+
func (s Scanner) Type() detectorspb.DetectorType {
101+
return detectorspb.DetectorType_Flexport
102+
}
103+
104+
func (s Scanner) Description() string {
105+
return "Flexport is a global logistics company that provides shipping, freight forwarding, and supply chain management services."
106+
}

‎pkg/detectors/flexport/flexport_test.go

Lines changed: 238 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,238 @@
1+
//go:build detectors
2+
// +build detectors
3+
4+
package flexport
5+
6+
import (
7+
"context"
8+
"fmt"
9+
"testing"
10+
"time"
11+
12+
"github.com/google/go-cmp/cmp"
13+
"github.com/google/go-cmp/cmp/cmpopts"
14+
15+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
16+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
17+
"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
18+
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
19+
)
20+
21+
func TestFlexport_Pattern(t *testing.T) {
22+
d := Scanner{}
23+
ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
24+
tests := []struct {
25+
name string
26+
input string
27+
want []string
28+
}{
29+
{
30+
name: "typical pattern",
31+
input: "flexport_token = 'shltm_ZnpDDh4AEj_n2WHHqjYErtv3ZGS0kH1bWVdl7V9D'",
32+
want: []string{"shltm_ZnpDDh4AEj_n2WHHqjYErtv3ZGS0kH1bWVdl7V9D"},
33+
},
34+
}
35+
36+
for _, test := range tests {
37+
t.Run(test.name, func(t *testing.T) {
38+
matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
39+
if len(matchedDetectors) == 0 {
40+
t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
41+
return
42+
}
43+
44+
results, err := d.FromData(context.Background(), false, []byte(test.input))
45+
if err != nil {
46+
t.Errorf("error = %v", err)
47+
return
48+
}
49+
50+
if len(results) != len(test.want) {
51+
if len(results) == 0 {
52+
t.Errorf("did not receive result")
53+
} else {
54+
t.Errorf("expected %d results, only received %d", len(test.want), len(results))
55+
}
56+
return
57+
}
58+
59+
actual := make(map[string]struct{}, len(results))
60+
for _, r := range results {
61+
if len(r.RawV2) > 0 {
62+
actual[string(r.RawV2)] = struct{}{}
63+
} else {
64+
actual[string(r.Raw)] = struct{}{}
65+
}
66+
}
67+
expected := make(map[string]struct{}, len(test.want))
68+
for _, v := range test.want {
69+
expected[v] = struct{}{}
70+
}
71+
72+
if diff := cmp.Diff(expected, actual); diff != "" {
73+
t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
74+
}
75+
})
76+
}
77+
}
78+
79+
func TestFlexport_FromChunk(t *testing.T) {
80+
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
81+
defer cancel()
82+
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
83+
if err != nil {
84+
t.Fatalf("could not get test secrets from GCP: %s", err)
85+
}
86+
secret := testSecrets.MustGetField("FLEXPORT")
87+
inactiveSecret := testSecrets.MustGetField("FLEXPORT_INACTIVE")
88+
secretNoPermissions := testSecrets.MustGetField("FLEXPORT_NO_PERMISSIONS")
89+
90+
type args struct {
91+
ctx context.Context
92+
data []byte
93+
verify bool
94+
}
95+
tests := []struct {
96+
name string
97+
s Scanner
98+
args args
99+
want []detectors.Result
100+
wantErr bool
101+
wantVerificationErr bool
102+
}{
103+
{
104+
name: "found, verified - with permissions",
105+
s: Scanner{},
106+
args: args{
107+
ctx: context.Background(),
108+
data: []byte(fmt.Sprintf("You can find a flexport secret %s within", secret)),
109+
verify: true,
110+
},
111+
want: []detectors.Result{
112+
{
113+
DetectorType: detectorspb.DetectorType_Flexport,
114+
Verified: true,
115+
},
116+
},
117+
wantErr: false,
118+
wantVerificationErr: false,
119+
},
120+
{
121+
name: "found, verified - without permissions",
122+
s: Scanner{},
123+
args: args{
124+
ctx: context.Background(),
125+
data: []byte(fmt.Sprintf("You can find a flexport secret %s within", secretNoPermissions)),
126+
verify: true,
127+
},
128+
want: []detectors.Result{
129+
{
130+
DetectorType: detectorspb.DetectorType_Flexport,
131+
Verified: true,
132+
},
133+
},
134+
wantErr: false,
135+
wantVerificationErr: false,
136+
},
137+
{
138+
name: "found, unverified",
139+
s: Scanner{},
140+
args: args{
141+
ctx: context.Background(),
142+
data: []byte(fmt.Sprintf("You can find a flexport secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
143+
verify: true,
144+
},
145+
want: []detectors.Result{
146+
{
147+
DetectorType: detectorspb.DetectorType_Flexport,
148+
Verified: false,
149+
},
150+
},
151+
wantErr: false,
152+
wantVerificationErr: false,
153+
},
154+
{
155+
name: "not found",
156+
s: Scanner{},
157+
args: args{
158+
ctx: context.Background(),
159+
data: []byte("You cannot find the secret within"),
160+
verify: true,
161+
},
162+
want: nil,
163+
wantErr: false,
164+
wantVerificationErr: false,
165+
},
166+
{
167+
name: "found, would be verified if not for timeout",
168+
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
169+
args: args{
170+
ctx: context.Background(),
171+
data: []byte(fmt.Sprintf("You can find a flexport secret %s within", secret)),
172+
verify: true,
173+
},
174+
want: []detectors.Result{
175+
{
176+
DetectorType: detectorspb.DetectorType_Flexport,
177+
Verified: false,
178+
},
179+
},
180+
wantErr: false,
181+
wantVerificationErr: true,
182+
},
183+
{
184+
name: "found, verified but unexpected api surface",
185+
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
186+
args: args{
187+
ctx: context.Background(),
188+
data: []byte(fmt.Sprintf("You can find a flexport secret %s within", secret)),
189+
verify: true,
190+
},
191+
want: []detectors.Result{
192+
{
193+
DetectorType: detectorspb.DetectorType_Flexport,
194+
Verified: false,
195+
},
196+
},
197+
wantErr: false,
198+
wantVerificationErr: true,
199+
},
200+
}
201+
for _, tt := range tests {
202+
t.Run(tt.name, func(t *testing.T) {
203+
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
204+
if (err != nil) != tt.wantErr {
205+
t.Errorf("Flexport.FromData() error = %v, wantErr %v", err, tt.wantErr)
206+
return
207+
}
208+
for i := range got {
209+
if len(got[i].Raw) == 0 {
210+
t.Fatalf("no raw secret present: \n %+v", got[i])
211+
}
212+
if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
213+
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
214+
}
215+
}
216+
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
217+
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
218+
t.Errorf("Flexport.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
219+
}
220+
})
221+
}
222+
}
223+
224+
func BenchmarkFromData(benchmark *testing.B) {
225+
ctx := context.Background()
226+
s := Scanner{}
227+
for name, data := range detectors.MustGetBenchmarkData() {
228+
benchmark.Run(name, func(b *testing.B) {
229+
b.ResetTimer()
230+
for n := 0; n < b.N; n++ {
231+
_, err := s.FromData(ctx, false, data)
232+
if err != nil {
233+
b.Fatal(err)
234+
}
235+
}
236+
})
237+
}
238+
}

‎pkg/engine/defaults/defaults.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -275,6 +275,7 @@ import (
275275
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/fixerio"
276276
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/flatio"
277277
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/fleetbase"
278+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/flexport"
278279
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/flickr"
279280
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/flightapi"
280281
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/flightlabs"
@@ -1102,6 +1103,7 @@ func buildDetectorList() []detectors.Detector {
11021103
&fixerio.Scanner{},
11031104
&flatio.Scanner{},
11041105
&fleetbase.Scanner{},
1106+
&flexport.Scanner{},
11051107
&flickr.Scanner{},
11061108
&flightapi.Scanner{},
11071109
&flightlabs.Scanner{},

0 commit comments

Comments
 (0)