Skip to content

Commit fa1514e

Browse files
nabeelalamnabeelalam
authored
Feature: Airtable OAuth Detector (#3868)
* added airtable oauth detector * updated regex pattern for airtable oauth * added comment on airtable oauth token regex
1 parent 26e4b24 commit fa1514e

File tree

6 files changed

+372
-7
lines changed

6 files changed

+372
-7
lines changed

‎pkg/detectors/airtableoauth/airtableoauth.go

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
package airtableoauth
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"io"
7+
"net/http"
8+
9+
regexp "github.com/wasilibs/go-re2"
10+
11+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
12+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
13+
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
14+
)
15+
16+
type Scanner struct {
17+
client *http.Client
18+
}
19+
20+
// Ensure the Scanner satisfies the interface at compile time.
21+
var _ detectors.Detector = (*Scanner)(nil)
22+
23+
var (
24+
defaultClient = common.SaneHttpClient()
25+
// The detector will attempt to match access tokens generated through the Airtable OAuth flow
26+
// Airtable OAuth does not support generating access tokens using client ID and key
27+
// Reference: https://airtable.com/developers/web/api/oauth-reference
28+
tokenPat = regexp.MustCompile(`\b([[:alnum:]]+\.v1\.[a-zA-Z0-9_-]+\.[a-f0-9]+)\b`)
29+
)
30+
31+
// Keywords are used for efficiently pre-filtering chunks.
32+
// Use identifiers in the secret preferably, or the provider name.
33+
func (s Scanner) Keywords() []string {
34+
return []string{"airtable"}
35+
}
36+
37+
// FromData will find and optionally verify AirtableOAuth secrets in a given set of bytes.
38+
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
39+
dataStr := string(data)
40+
41+
uniqueMatches := make(map[string]struct{})
42+
for _, match := range tokenPat.FindAllStringSubmatch(dataStr, -1) {
43+
uniqueMatches[match[1]] = struct{}{}
44+
}
45+
46+
for match := range uniqueMatches {
47+
s1 := detectors.Result{
48+
DetectorType: detectorspb.DetectorType_AirtableOAuth,
49+
Raw: []byte(match),
50+
}
51+
52+
if verify {
53+
client := s.client
54+
if client == nil {
55+
client = defaultClient
56+
}
57+
58+
isVerified, extraData, verificationErr := verifyMatch(ctx, client, match)
59+
s1.Verified = isVerified
60+
s1.ExtraData = extraData
61+
s1.SetVerificationError(verificationErr, match)
62+
}
63+
64+
results = append(results, s1)
65+
}
66+
67+
return
68+
}
69+
70+
func verifyMatch(ctx context.Context, client *http.Client, token string) (bool, map[string]string, error) {
71+
endpoint := "https://api.airtable.com/v0/meta/whoami"
72+
req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
73+
if err != nil {
74+
return false, nil, nil
75+
}
76+
77+
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))
78+
res, err := client.Do(req)
79+
if err != nil {
80+
return false, nil, err
81+
}
82+
defer func() {
83+
_, _ = io.Copy(io.Discard, res.Body)
84+
_ = res.Body.Close()
85+
}()
86+
87+
switch res.StatusCode {
88+
case http.StatusOK:
89+
return true, nil, nil
90+
case http.StatusUnauthorized:
91+
// The secret is determinately not verified (nothing to do)
92+
return false, nil, nil
93+
default:
94+
return false, nil, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
95+
}
96+
}
97+
98+
func (s Scanner) Type() detectorspb.DetectorType {
99+
return detectorspb.DetectorType_AirtableOAuth
100+
}
101+
102+
func (s Scanner) Description() string {
103+
return "Airtable is a cloud collaboration service that offers database-like features. Airtable OAuth tokens can be used to access and modify data within Airtable bases."
104+
}

‎pkg/detectors/airtableoauth/airtableoauth_integration_test.go

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
//go:build detectors
2+
// +build detectors
3+
4+
package airtableoauth
5+
6+
import (
7+
"context"
8+
"fmt"
9+
"testing"
10+
"time"
11+
12+
"github.com/google/go-cmp/cmp"
13+
"github.com/google/go-cmp/cmp/cmpopts"
14+
15+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
16+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
17+
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
18+
)
19+
20+
// TestAirtableoauth_FromChunk verifies the validity of an Airtable OAuth token
21+
// Note: The token validity test relies on an access token stored in the GCP secret manager.
22+
// Since Airtable OAuth tokens expire after 60 minutes, this test will eventually fail once the token becomes invalid.
23+
// The official guide linked below can be followed in order to generate a new valid access token:
24+
// https://airtable.com/developers/web/api/oauth-reference
25+
26+
func TestAirtableoauth_FromChunk(t *testing.T) {
27+
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
28+
defer cancel()
29+
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
30+
if err != nil {
31+
t.Fatalf("could not get test secrets from GCP: %s", err)
32+
}
33+
secret := testSecrets.MustGetField("AIRTABLEOAUTH")
34+
inactiveSecret := testSecrets.MustGetField("AIRTABLEOAUTH_INACTIVE")
35+
36+
type args struct {
37+
ctx context.Context
38+
data []byte
39+
verify bool
40+
}
41+
tests := []struct {
42+
name string
43+
s Scanner
44+
args args
45+
want []detectors.Result
46+
wantErr bool
47+
wantVerificationErr bool
48+
}{
49+
{
50+
name: "found, verified",
51+
s: Scanner{},
52+
args: args{
53+
ctx: context.Background(),
54+
data: []byte(fmt.Sprintf("You can find a airtableoauth secret %s within", secret)),
55+
verify: true,
56+
},
57+
want: []detectors.Result{
58+
{
59+
DetectorType: detectorspb.DetectorType_AirtableOAuth,
60+
Verified: true,
61+
},
62+
},
63+
wantErr: false,
64+
wantVerificationErr: false,
65+
},
66+
{
67+
name: "found, unverified",
68+
s: Scanner{},
69+
args: args{
70+
ctx: context.Background(),
71+
data: []byte(fmt.Sprintf("You can find a airtableoauth secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
72+
verify: true,
73+
},
74+
want: []detectors.Result{
75+
{
76+
DetectorType: detectorspb.DetectorType_AirtableOAuth,
77+
Verified: false,
78+
},
79+
},
80+
wantErr: false,
81+
wantVerificationErr: false,
82+
},
83+
{
84+
name: "not found",
85+
s: Scanner{},
86+
args: args{
87+
ctx: context.Background(),
88+
data: []byte("You cannot find the secret within"),
89+
verify: true,
90+
},
91+
want: nil,
92+
wantErr: false,
93+
wantVerificationErr: false,
94+
},
95+
{
96+
name: "found, would be verified if not for timeout",
97+
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
98+
args: args{
99+
ctx: context.Background(),
100+
data: []byte(fmt.Sprintf("You can find a airtableoauth secret %s within", secret)),
101+
verify: true,
102+
},
103+
want: []detectors.Result{
104+
{
105+
DetectorType: detectorspb.DetectorType_AirtableOAuth,
106+
Verified: false,
107+
},
108+
},
109+
wantErr: false,
110+
wantVerificationErr: true,
111+
},
112+
{
113+
name: "found, verified but unexpected api surface",
114+
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
115+
args: args{
116+
ctx: context.Background(),
117+
data: []byte(fmt.Sprintf("You can find a airtableoauth secret %s within", secret)),
118+
verify: true,
119+
},
120+
want: []detectors.Result{
121+
{
122+
DetectorType: detectorspb.DetectorType_AirtableOAuth,
123+
Verified: false,
124+
},
125+
},
126+
wantErr: false,
127+
wantVerificationErr: true,
128+
},
129+
}
130+
for _, tt := range tests {
131+
t.Run(tt.name, func(t *testing.T) {
132+
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
133+
if (err != nil) != tt.wantErr {
134+
t.Errorf("Airtableoauth.FromData() error = %v, wantErr %v", err, tt.wantErr)
135+
return
136+
}
137+
for i := range got {
138+
if len(got[i].Raw) == 0 {
139+
t.Fatalf("no raw secret present: \n %+v", got[i])
140+
}
141+
if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
142+
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
143+
}
144+
}
145+
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
146+
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
147+
t.Errorf("Airtableoauth.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
148+
}
149+
})
150+
}
151+
}
152+
153+
func BenchmarkFromData(benchmark *testing.B) {
154+
ctx := context.Background()
155+
s := Scanner{}
156+
for name, data := range detectors.MustGetBenchmarkData() {
157+
benchmark.Run(name, func(b *testing.B) {
158+
b.ResetTimer()
159+
for n := 0; n < b.N; n++ {
160+
_, err := s.FromData(ctx, false, data)
161+
if err != nil {
162+
b.Fatal(err)
163+
}
164+
}
165+
})
166+
}
167+
}

‎pkg/detectors/airtableoauth/airtableoauth_test.go

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
package airtableoauth
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"testing"
7+
8+
"github.com/google/go-cmp/cmp"
9+
10+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
11+
"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
12+
)
13+
14+
var (
15+
validPattern1 = "oaajtCy2lVMUN1Cm5.v1.eyJ1c2VySWQiOiJ1c3JjQ09QVlJudGlrU1lzdyIsImV4cGlyZXNBdCI6IjIwMjUtMDItMDNUMTk6NTY6MzcuMDAwWiIsIm9hdXRoQXBwbGljYXRpb25JZCI6Im9hcG14aXcyUlRrVGlzcHJIIiwic2VjcmV0IjoiMzczNThlNzdlZjlhMjljY2Q5MWIwNmNlNTdkZDYxNDg0MWVmNmIyOWYwYjQ5ZWE0MTMxZGI4NzBkNTAzYTE1NyJ9.0d67c8b334048135a93615610445e4aa90c6af6222392b49eea9419e1d6717d0"
16+
validPattern2 = "oaaRYiYSlTFXZzxDM.v1.eyJ1c2VySWQiOiJ1c3JjQ09QVlJudGlrU1lzdyIsIm9hdXRoQXBwbGljYXRpb25JZCI6Im9hcG14aXcyUlRrVGlzcHJIIiwiZXhwaXJlc0F0IjoiMjAyNS0wMS0yOVQwMDowMTo0NC4wMDBaIiwic2VjcmV0IjoiZjYyOWE1MWVkM2M0ZjU5ODlmOTcyMDU1ZjkwODk3NDA4NmU0NjQxY2JhODU5Y2FhZTJkZjliMWQwODg0ZjIzMiJ9.27a8998029ac9bdd599b435572821dcb63c60cbf62b9cb2ba2a73511e5553d66"
17+
invalidPattern = "oaaRYiYSlTFXZzxDM.v2.eyJ1c2VySWQiOiJ1c3JjQ09QVlJudGlrU1lzdyIsIm9hdXRoQXBwbGljYXRpb25JZCI6Im9hcG14aXcyUlRrVGlzcHJIIiwiZXhwaXJlc0F0IjoiMjAyNS0wMS0yOVQwMDowMTo0NC4wMDBaIiwic2VjcmV0IjoiZjYyOWE1MWVkM2M0ZjU5ODlmOTcyMDU1ZjkwODk3NDA4NmU0NjQxY2JhODU5Y2FhZTJkZjliMWQwODg0ZjIzMiJ9.27a8998029ac9bdd599b435572821dcb63c60cbf62b9cb2ba2a73511e5553d66"
18+
)
19+
20+
func TestAirtableoauth_Pattern(t *testing.T) {
21+
d := Scanner{}
22+
ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
23+
tests := []struct {
24+
name string
25+
input string
26+
want []string
27+
}{
28+
{
29+
name: "typical pattern",
30+
input: fmt.Sprintf("airtable token = '%s'", validPattern1),
31+
want: []string{validPattern1},
32+
},
33+
{
34+
name: "finds all matches",
35+
input: fmt.Sprintf(`airtable token 1 = '%s'
36+
airtabl token 2 = '%s'`, validPattern1, validPattern2),
37+
want: []string{validPattern1, validPattern2},
38+
},
39+
{
40+
name: "invalid pattern",
41+
input: fmt.Sprintf("airtable token = '%s'", invalidPattern),
42+
want: []string{},
43+
},
44+
}
45+
46+
for _, test := range tests {
47+
t.Run(test.name, func(t *testing.T) {
48+
matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
49+
if len(matchedDetectors) == 0 {
50+
t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
51+
return
52+
}
53+
54+
results, err := d.FromData(context.Background(), false, []byte(test.input))
55+
if err != nil {
56+
t.Errorf("error = %v", err)
57+
return
58+
}
59+
60+
if len(results) != len(test.want) {
61+
if len(results) == 0 {
62+
t.Errorf("did not receive result")
63+
} else {
64+
t.Errorf("expected %d results, only received %d", len(test.want), len(results))
65+
}
66+
return
67+
}
68+
69+
actual := make(map[string]struct{}, len(results))
70+
for _, r := range results {
71+
if len(r.RawV2) > 0 {
72+
actual[string(r.RawV2)] = struct{}{}
73+
} else {
74+
actual[string(r.Raw)] = struct{}{}
75+
}
76+
}
77+
expected := make(map[string]struct{}, len(test.want))
78+
for _, v := range test.want {
79+
expected[v] = struct{}{}
80+
}
81+
82+
if diff := cmp.Diff(expected, actual); diff != "" {
83+
t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
84+
}
85+
})
86+
}
87+
}

‎pkg/engine/defaults/defaults.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import (
1414
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/airbrakeuserkey"
1515
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/airship"
1616
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/airtableapikey"
17+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/airtableoauth"
1718
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/airvisual"
1819
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/aiven"
1920
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/alchemy"
@@ -837,6 +838,7 @@ func buildDetectorList() []detectors.Detector {
837838
&airbrakeuserkey.Scanner{},
838839
&airship.Scanner{},
839840
&airtableapikey.Scanner{},
841+
&airtableoauth.Scanner{},
840842
&airvisual.Scanner{},
841843
&aiven.Scanner{},
842844
&alchemy.Scanner{},

0 commit comments

Comments
 (0)