Skip to content

Commit d8658ce

Browse files
SyedAliHamadamanfcp
SyedAliHamad
and
amanfcp
authored
Oss 133 new detector vault approle auth for hashicorp (#4362)
* add detector for hashicorp vault auth * resolve comments and update test cases * add indefinite response handling * resolve comments * update error response * resolve merge issues * follow code convention --------- Co-authored-by: Amaan Ullah <[email protected]>
1 parent 31d1b13 commit d8658ce

File tree

6 files changed

+510
-7
lines changed

6 files changed

+510
-7
lines changed

‎pkg/detectors/hashicorpvaultauth/hashicorpvaultauth.go

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
package hashicorpvaultauth
2+
3+
import (
4+
"bytes"
5+
"context"
6+
"encoding/json"
7+
"fmt"
8+
"io"
9+
"net/http"
10+
"strings"
11+
12+
regexp "github.com/wasilibs/go-re2"
13+
14+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
15+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
16+
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
17+
)
18+
19+
type Scanner struct {
20+
client *http.Client
21+
}
22+
23+
// Ensure the Scanner satisfies the interface at compile time.
24+
var _ detectors.Detector = (*Scanner)(nil)
25+
26+
var (
27+
defaultClient = common.SaneHttpClient()
28+
29+
roleIdPat = regexp.MustCompile(detectors.PrefixRegex([]string{"role"}) + `\b([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\b`)
30+
31+
secretIdPat = regexp.MustCompile(detectors.PrefixRegex([]string{"secret"}) + `\b([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})\b`)
32+
33+
// Vault URL pattern - HashiCorp Cloud or any HTTPS/HTTP Vault endpoint
34+
vaultUrlPat = regexp.MustCompile(`(https?:\/\/[^\s\/]*\.hashicorp\.cloud(?::\d+)?)(?:\/[^\s]*)?`)
35+
)
36+
37+
// Keywords are used for efficiently pre-filtering chunks.
38+
func (s Scanner) Keywords() []string {
39+
return []string{"hashicorp"}
40+
}
41+
42+
// FromData will find and optionally verify HashiCorp Vault AppRole secrets in a given set of bytes.
43+
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
44+
dataStr := string(data)
45+
46+
var uniqueRoleIds = make(map[string]struct{})
47+
for _, match := range roleIdPat.FindAllStringSubmatch(dataStr, -1) {
48+
roleId := strings.TrimSpace(match[1])
49+
uniqueRoleIds[roleId] = struct{}{}
50+
}
51+
52+
var uniqueSecretIds = make(map[string]struct{})
53+
for _, match := range secretIdPat.FindAllStringSubmatch(dataStr, -1) {
54+
secretId := strings.TrimSpace(match[1])
55+
uniqueSecretIds[secretId] = struct{}{}
56+
}
57+
58+
var uniqueVaultUrls = make(map[string]struct{})
59+
for _, match := range vaultUrlPat.FindAllString(dataStr, -1) {
60+
url := strings.TrimSpace(match)
61+
uniqueVaultUrls[url] = struct{}{}
62+
}
63+
64+
// If no names or secrets found, return empty results
65+
if len(uniqueRoleIds) == 0 || len(uniqueSecretIds) == 0 || len(uniqueVaultUrls) == 0 {
66+
return results, nil
67+
}
68+
69+
// create combination results that can be verified
70+
for roleId := range uniqueRoleIds {
71+
for secretId := range uniqueSecretIds {
72+
for vaultUrl := range uniqueVaultUrls {
73+
s1 := detectors.Result{
74+
DetectorType: detectorspb.DetectorType_HashiCorpVaultAuth,
75+
Raw: []byte(secretId),
76+
RawV2: []byte(fmt.Sprintf("%s:%s", roleId, secretId)),
77+
ExtraData: map[string]string{
78+
"URL": vaultUrl,
79+
},
80+
}
81+
82+
if verify {
83+
client := s.client
84+
if client == nil {
85+
client = defaultClient
86+
}
87+
88+
isVerified, verificationErr := verifyMatch(ctx, client, roleId, secretId, vaultUrl)
89+
s1.Verified = isVerified
90+
s1.SetVerificationError(verificationErr, roleId, secretId, vaultUrl)
91+
}
92+
results = append(results, s1)
93+
}
94+
}
95+
}
96+
return results, nil
97+
}
98+
99+
func verifyMatch(ctx context.Context, client *http.Client, roleId, secretId, vaultUrl string) (bool, error) {
100+
payload := map[string]string{
101+
"role_id": roleId,
102+
"secret_id": secretId,
103+
}
104+
105+
jsonPayload, err := json.Marshal(payload)
106+
if err != nil {
107+
return false, err
108+
}
109+
110+
req, err := http.NewRequestWithContext(ctx, http.MethodPost, vaultUrl+"/v1/auth/approle/login", bytes.NewBuffer(jsonPayload))
111+
if err != nil {
112+
return false, err
113+
}
114+
req.Header.Set("Content-Type", "application/json")
115+
req.Header.Set("X-Vault-Namespace", "admin")
116+
117+
resp, err := client.Do(req)
118+
if err != nil {
119+
return false, err
120+
}
121+
122+
defer func() {
123+
_, _ = io.Copy(io.Discard, resp.Body)
124+
_ = resp.Body.Close()
125+
}()
126+
127+
switch resp.StatusCode {
128+
case http.StatusOK:
129+
return true, nil
130+
case http.StatusBadRequest:
131+
body, err := io.ReadAll(resp.Body)
132+
if err != nil {
133+
return false, err
134+
}
135+
136+
if strings.Contains(string(body), "invalid role or secret ID") {
137+
return false, nil
138+
} else {
139+
return false, fmt.Errorf("unexpected HTTP response status %d", resp.StatusCode)
140+
}
141+
default:
142+
return false, fmt.Errorf("unexpected HTTP response status %d", resp.StatusCode)
143+
}
144+
}
145+
146+
func (s Scanner) Type() detectorspb.DetectorType {
147+
return detectorspb.DetectorType_HashiCorpVaultAuth
148+
}
149+
150+
func (s Scanner) Description() string {
151+
return "HashiCorp Vault AppRole authentication method uses role_id and secret_id for machine-to-machine authentication. These credentials can be used to authenticate with Vault and obtain tokens for accessing secrets."
152+
}

‎pkg/detectors/hashicorpvaultauth/hashicorpvaultauth_integration_test.go

Lines changed: 181 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,181 @@
1+
//go:build detectors
2+
// +build detectors
3+
4+
package hashicorpvaultauth
5+
6+
import (
7+
"context"
8+
"fmt"
9+
"testing"
10+
"time"
11+
12+
"github.com/google/go-cmp/cmp"
13+
"github.com/google/go-cmp/cmp/cmpopts"
14+
15+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
16+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
17+
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
18+
)
19+
20+
func TestHashiCorpVaultAuth_FromChunk(t *testing.T) {
21+
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
22+
defer cancel()
23+
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors6")
24+
if err != nil {
25+
t.Fatalf("could not get test secrets from GCP: %s", err)
26+
}
27+
roleId := testSecrets.MustGetField("HASHICORPVAULTAUTH_ROLE_ID")
28+
secretId := testSecrets.MustGetField("HASHICORPVAULTAUTH_SECRET_ID")
29+
inactiveRoleId := testSecrets.MustGetField("HASHICORPVAULTAUTH_ROLE_ID_INACTIVE")
30+
inactiveSecretId := testSecrets.MustGetField("HASHICORPVAULTAUTH_SECRET_ID_INACTIVE")
31+
vaultUrl := testSecrets.MustGetField("HASHICORPVAULTAUTH_URL")
32+
33+
type args struct {
34+
ctx context.Context
35+
data []byte
36+
verify bool
37+
}
38+
tests := []struct {
39+
name string
40+
s Scanner
41+
args args
42+
want []detectors.Result
43+
wantErr bool
44+
wantVerificationErr bool
45+
}{
46+
{
47+
name: "found, unverified - complete set with invalid credentials",
48+
s: Scanner{},
49+
args: args{
50+
ctx: context.Background(),
51+
data: []byte(fmt.Sprintf("hashicorp config:\nrole_id: %s\nsecret_id: %s\nvault_url: %s", inactiveRoleId, inactiveSecretId, vaultUrl)),
52+
verify: true,
53+
},
54+
want: []detectors.Result{
55+
{
56+
DetectorType: detectorspb.DetectorType_HashiCorpVaultAuth,
57+
Verified: false,
58+
VerificationFromCache: false,
59+
Raw: []byte(inactiveSecretId),
60+
RawV2: []byte(fmt.Sprintf("%s:%s", inactiveRoleId, inactiveSecretId)),
61+
ExtraData: map[string]string{
62+
"URL": vaultUrl,
63+
},
64+
StructuredData: nil,
65+
},
66+
},
67+
wantErr: false,
68+
wantVerificationErr: false,
69+
},
70+
{
71+
name: "found, verified - complete set with valid credentials",
72+
s: Scanner{},
73+
args: args{
74+
ctx: context.Background(),
75+
data: []byte(fmt.Sprintf("hashicorp config:\nrole_id: %s\nsecret_id: %s\nvault_url: %s", roleId, secretId, vaultUrl)),
76+
verify: true,
77+
},
78+
want: []detectors.Result{
79+
{
80+
DetectorType: detectorspb.DetectorType_HashiCorpVaultAuth,
81+
Verified: true,
82+
VerificationFromCache: false,
83+
Raw: []byte(secretId),
84+
RawV2: []byte(fmt.Sprintf("%s:%s", roleId, secretId)),
85+
ExtraData: map[string]string{
86+
"URL": vaultUrl,
87+
},
88+
StructuredData: nil,
89+
},
90+
},
91+
wantErr: false,
92+
wantVerificationErr: false,
93+
},
94+
{
95+
name: "found, incomplete set - credentials without vault url",
96+
s: Scanner{},
97+
args: args{
98+
ctx: context.Background(),
99+
data: []byte(fmt.Sprintf("vault config:\nrole_id: %s\nsecret_id: %s", roleId, secretId)),
100+
verify: true,
101+
},
102+
want: nil,
103+
wantErr: false,
104+
wantVerificationErr: false,
105+
},
106+
{
107+
name: "found, incomplete set - only role_id",
108+
s: Scanner{},
109+
args: args{
110+
ctx: context.Background(),
111+
data: []byte(fmt.Sprintf("vault role_id: %s", roleId)),
112+
verify: true,
113+
},
114+
want: nil,
115+
wantErr: false,
116+
wantVerificationErr: false,
117+
},
118+
{
119+
name: "found, incomplete set - only secret_id",
120+
s: Scanner{},
121+
args: args{
122+
ctx: context.Background(),
123+
data: []byte(fmt.Sprintf("vault secret_id: %s", secretId)),
124+
verify: true,
125+
},
126+
want: nil,
127+
wantErr: false,
128+
wantVerificationErr: false,
129+
},
130+
{
131+
name: "not found - no vault context",
132+
s: Scanner{},
133+
args: args{
134+
ctx: context.Background(),
135+
data: []byte("You cannot find the secret within"),
136+
verify: true,
137+
},
138+
want: nil,
139+
wantErr: false,
140+
wantVerificationErr: false,
141+
},
142+
}
143+
for _, tt := range tests {
144+
t.Run(tt.name, func(t *testing.T) {
145+
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
146+
if (err != nil) != tt.wantErr {
147+
t.Errorf("HashiCorpVaultAuth.FromData() error = %v, wantErr %v", err, tt.wantErr)
148+
return
149+
}
150+
for i := range got {
151+
if len(got[i].Raw) == 0 {
152+
t.Fatalf("no raw secret present: \n %+v", got[i])
153+
}
154+
if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
155+
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
156+
}
157+
}
158+
// Fix: Ignore ALL unexported fields using cmpopts.IgnoreUnexported
159+
ignoreOpts := cmpopts.IgnoreUnexported(detectors.Result{})
160+
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
161+
t.Errorf("HashiCorpVaultAuth.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
162+
}
163+
})
164+
}
165+
}
166+
167+
func BenchmarkFromData(benchmark *testing.B) {
168+
ctx := context.Background()
169+
s := Scanner{}
170+
for name, data := range detectors.MustGetBenchmarkData() {
171+
benchmark.Run(name, func(b *testing.B) {
172+
b.ResetTimer()
173+
for n := 0; n < b.N; n++ {
174+
_, err := s.FromData(ctx, false, data)
175+
if err != nil {
176+
b.Fatal(err)
177+
}
178+
}
179+
})
180+
}
181+
}

0 commit comments

Comments
 (0)