Skip to content

Commit 60e359b

Browse files
amanfcpamanfcp
authored
Netlify Detector Version 2 implementation (#4091)
* task: implementation of netlify v2 init * test: ran and modified tests
1 parent 7008ad2 commit 60e359b

File tree

7 files changed

+314
-3
lines changed

7 files changed

+314
-3
lines changed

‎pkg/detectors/netlify/netlify.go renamed to ‎pkg/detectors/netlify/v1/netlify_v1.go

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,12 @@ package netlify
33
import (
44
"context"
55
"fmt"
6-
regexp "github.com/wasilibs/go-re2"
76
"net/http"
7+
"strconv"
88
"strings"
99

10+
regexp "github.com/wasilibs/go-re2"
11+
1012
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
1113
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
1214
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@@ -16,13 +18,16 @@ type Scanner struct{}
1618

1719
// Ensure the Scanner satisfies the interface at compile time.
1820
var _ detectors.Detector = (*Scanner)(nil)
21+
var _ detectors.Versioner = (*Scanner)(nil)
1922

2023
var (
2124
client = common.SaneHttpClient()
2225

2326
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"netlify"}) + `\b([A-Za-z0-9_-]{43,45})\b`)
2427
)
2528

29+
func (Scanner) Version() int { return 1 }
30+
2631
// Keywords are used for efficiently pre-filtering chunks.
2732
// Use identifiers in the secret preferably, or the provider name.
2833
func (s Scanner) Keywords() []string {
@@ -44,6 +49,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
4449
}
4550
s1.ExtraData = map[string]string{
4651
"rotation_guide": "https://howtorotate.com/docs/tutorials/netlify/",
52+
"version": strconv.Itoa(s.Version()),
4753
}
4854

4955
if verify {

‎pkg/detectors/netlify/netlify_integration_test.go renamed to ‎pkg/detectors/netlify/v1/netlify_v1_integration_test.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ func TestNetlify_FromChunk(t *testing.T) {
5252
Verified: true,
5353
ExtraData: map[string]string{
5454
"rotation_guide": "https://howtorotate.com/docs/tutorials/netlify/",
55+
"version": "1",
5556
},
5657
},
5758
},
@@ -71,6 +72,7 @@ func TestNetlify_FromChunk(t *testing.T) {
7172
Verified: false,
7273
ExtraData: map[string]string{
7374
"rotation_guide": "https://howtorotate.com/docs/tutorials/netlify/",
75+
"version": "1",
7476
},
7577
},
7678
},

‎pkg/detectors/netlify/netlify_test.go renamed to ‎pkg/detectors/netlify/v1/netlify_v1_test.go

File renamed without changes.

‎pkg/detectors/netlify/v2/netlify_v2.go

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
package netlify
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"net/http"
7+
"strconv"
8+
"strings"
9+
10+
regexp "github.com/wasilibs/go-re2"
11+
12+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
13+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
14+
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
15+
)
16+
17+
type Scanner struct{}
18+
19+
// Ensure the Scanner satisfies the interface at compile time.
20+
var _ detectors.Detector = (*Scanner)(nil)
21+
var _ detectors.Versioner = (*Scanner)(nil)
22+
23+
var (
24+
client = common.SaneHttpClient()
25+
26+
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"netlify"}) + `\b(nfp_[a-zA-Z0-9_]{36})\b`)
27+
)
28+
29+
func (Scanner) Version() int { return 2 }
30+
31+
// Keywords are used for efficiently pre-filtering chunks.
32+
// Use identifiers in the secret preferably, or the provider name.
33+
func (s Scanner) Keywords() []string {
34+
return []string{"netlify"}
35+
}
36+
37+
// FromData will find and optionally verify Netlify secrets in a given set of bytes.
38+
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
39+
dataStr := string(data)
40+
41+
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
42+
43+
for _, match := range matches {
44+
resMatch := strings.TrimSpace(match[1])
45+
46+
s1 := detectors.Result{
47+
DetectorType: detectorspb.DetectorType_Netlify,
48+
Raw: []byte(resMatch),
49+
}
50+
s1.ExtraData = map[string]string{
51+
"rotation_guide": "https://howtorotate.com/docs/tutorials/netlify/",
52+
"version": strconv.Itoa(s.Version()),
53+
}
54+
55+
if verify {
56+
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.netlify.com/api/v1/sites", nil)
57+
if err != nil {
58+
continue
59+
}
60+
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
61+
res, err := client.Do(req)
62+
if err == nil {
63+
defer res.Body.Close()
64+
if res.StatusCode >= 200 && res.StatusCode < 300 {
65+
s1.Verified = true
66+
}
67+
}
68+
}
69+
70+
results = append(results, s1)
71+
}
72+
73+
return results, nil
74+
}
75+
76+
func (s Scanner) Type() detectorspb.DetectorType {
77+
return detectorspb.DetectorType_Netlify
78+
}
79+
80+
func (s Scanner) Description() string {
81+
return "Netlify is a cloud platform for web developers that provides hosting and serverless backend services for web applications and static websites. Netlify API keys can be used to manage sites, deploy applications, and access various services offered by the platform."
82+
}

‎pkg/detectors/netlify/v2/netlify_v2_integration_test.go

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
//go:build detectors
2+
// +build detectors
3+
4+
package netlify
5+
6+
import (
7+
"context"
8+
"fmt"
9+
"testing"
10+
"time"
11+
12+
"github.com/kylelemons/godebug/pretty"
13+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
14+
15+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
16+
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
17+
)
18+
19+
func TestNetlify_FromChunk(t *testing.T) {
20+
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
21+
defer cancel()
22+
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
23+
if err != nil {
24+
t.Fatalf("could not get test secrets from GCP: %s", err)
25+
}
26+
secret := testSecrets.MustGetField("NETLIFY_TOKEN")
27+
inactiveSecret := testSecrets.MustGetField("NETLIFY_INACTIVE")
28+
29+
type args struct {
30+
ctx context.Context
31+
data []byte
32+
verify bool
33+
}
34+
tests := []struct {
35+
name string
36+
s Scanner
37+
args args
38+
want []detectors.Result
39+
wantErr bool
40+
}{
41+
{
42+
name: "found, verified",
43+
s: Scanner{},
44+
args: args{
45+
ctx: context.Background(),
46+
data: []byte(fmt.Sprintf("You can find a netlify secret %s within", secret)),
47+
verify: true,
48+
},
49+
want: []detectors.Result{
50+
{
51+
DetectorType: detectorspb.DetectorType_Netlify,
52+
Verified: true,
53+
ExtraData: map[string]string{
54+
"rotation_guide": "https://howtorotate.com/docs/tutorials/netlify/",
55+
"version": "2",
56+
},
57+
},
58+
},
59+
wantErr: false,
60+
},
61+
{
62+
name: "found, unverified",
63+
s: Scanner{},
64+
args: args{
65+
ctx: context.Background(),
66+
data: []byte(fmt.Sprintf("You can find a netlify secret %s within but unverified", inactiveSecret)),
67+
verify: true,
68+
},
69+
want: []detectors.Result{
70+
{
71+
DetectorType: detectorspb.DetectorType_Netlify,
72+
Verified: false,
73+
ExtraData: map[string]string{
74+
"rotation_guide": "https://howtorotate.com/docs/tutorials/netlify/",
75+
"version": "2",
76+
},
77+
},
78+
},
79+
wantErr: false,
80+
},
81+
{
82+
name: "not found",
83+
s: Scanner{},
84+
args: args{
85+
ctx: context.Background(),
86+
data: []byte("You cannot find the secret within"),
87+
verify: true,
88+
},
89+
want: nil,
90+
wantErr: false,
91+
},
92+
}
93+
for _, tt := range tests {
94+
t.Run(tt.name, func(t *testing.T) {
95+
s := Scanner{}
96+
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
97+
if (err != nil) != tt.wantErr {
98+
t.Errorf("Netlify.FromData() error = %v, wantErr %v", err, tt.wantErr)
99+
return
100+
}
101+
for i := range got {
102+
if len(got[i].Raw) == 0 {
103+
t.Fatalf("no raw secret present: \n %+v", got[i])
104+
}
105+
got[i].Raw = nil
106+
}
107+
if diff := pretty.Compare(got, tt.want); diff != "" {
108+
t.Errorf("Netlify.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
109+
}
110+
})
111+
}
112+
}
113+
114+
func BenchmarkFromData(benchmark *testing.B) {
115+
ctx := context.Background()
116+
s := Scanner{}
117+
for name, data := range detectors.MustGetBenchmarkData() {
118+
benchmark.Run(name, func(b *testing.B) {
119+
b.ResetTimer()
120+
for n := 0; n < b.N; n++ {
121+
_, err := s.FromData(ctx, false, data)
122+
if err != nil {
123+
b.Fatal(err)
124+
}
125+
}
126+
})
127+
}
128+
}

‎pkg/detectors/netlify/v2/netlify_v2_test.go

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
package netlify
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"testing"
7+
8+
"github.com/google/go-cmp/cmp"
9+
10+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
11+
"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
12+
)
13+
14+
var (
15+
validPattern = "nfp_gsTPZr8vTqmit69DY8u8es68N3XvxqW20330"
16+
invalidPattern = "nfp_?sTPZr8vTqmit69DY8u8es68N3XvxqW20330"
17+
keyword = "netlify"
18+
)
19+
20+
func TestNetlify_Pattern(t *testing.T) {
21+
d := Scanner{}
22+
ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
23+
tests := []struct {
24+
name string
25+
input string
26+
want []string
27+
}{
28+
{
29+
name: "valid pattern - with keyword netlify",
30+
input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
31+
want: []string{validPattern},
32+
},
33+
{
34+
name: "valid pattern - ignore duplicate",
35+
input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
36+
want: []string{validPattern},
37+
},
38+
{
39+
name: "valid pattern - key out of prefix range",
40+
input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
41+
want: []string{},
42+
},
43+
{
44+
name: "invalid pattern",
45+
input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
46+
want: []string{},
47+
},
48+
}
49+
50+
for _, test := range tests {
51+
t.Run(test.name, func(t *testing.T) {
52+
matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
53+
if len(matchedDetectors) == 0 {
54+
t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
55+
return
56+
}
57+
58+
results, err := d.FromData(context.Background(), false, []byte(test.input))
59+
if err != nil {
60+
t.Errorf("error = %v", err)
61+
return
62+
}
63+
64+
if len(results) != len(test.want) {
65+
if len(results) == 0 {
66+
t.Errorf("did not receive result")
67+
} else {
68+
t.Errorf("expected %d results, only received %d", len(test.want), len(results))
69+
}
70+
return
71+
}
72+
73+
actual := make(map[string]struct{}, len(results))
74+
for _, r := range results {
75+
if len(r.RawV2) > 0 {
76+
actual[string(r.RawV2)] = struct{}{}
77+
} else {
78+
actual[string(r.Raw)] = struct{}{}
79+
}
80+
}
81+
expected := make(map[string]struct{}, len(test.want))
82+
for _, v := range test.want {
83+
expected[v] = struct{}{}
84+
}
85+
86+
if diff := cmp.Diff(expected, actual); diff != "" {
87+
t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
88+
}
89+
})
90+
}
91+
}

‎pkg/engine/defaults/defaults.go

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -468,7 +468,8 @@ import (
468468
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/myfreshworks"
469469
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/myintervals"
470470
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/nethunt"
471-
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/netlify"
471+
netlifyv1 "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/netlify/v1"
472+
netlifyv2 "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/netlify/v2"
472473
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/netsuite"
473474
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/neutrinoapi"
474475
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/newrelicpersonalapikey"
@@ -1320,7 +1321,8 @@ func buildDetectorList() []detectors.Detector {
13201321
&myintervals.Scanner{},
13211322
// &nasdaqdatalink.Scanner{},
13221323
&nethunt.Scanner{},
1323-
&netlify.Scanner{},
1324+
&netlifyv1.Scanner{},
1325+
&netlifyv2.Scanner{},
13241326
&netsuite.Scanner{},
13251327
&neutrinoapi.Scanner{},
13261328
&newrelicpersonalapikey.Scanner{},

0 commit comments

Comments
 (0)