Skip to content

[Detector] - Restore and Refactor Detectors starting with *CA* #4315

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 8 commits into
base: main
Choose a base branch
from
Open

[Detector] - Restore and Refactor Detectors starting with *CA* #4315

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
674ad9a
Refactor verification logic across multiple detectors to use a unifie…
amanfcp Jul 14, 2025
19fd5ee
Remove debug print statement from `verifyMatch` function in cashboard…
amanfcp Jul 14, 2025
a546d8d
Merge branch 'main' into task/restore_and_refactor_c_letter_detectors
shahzadhaider1 Jul 17, 2025
e8f2a43
Refactor error handling in `verifyMatch` functions across multiple de…
amanfcp Jul 18, 2025
e3f4173
cannyio - directly check in body if it contains invalid api key
amanfcp Jul 18, 2025
0a8b805
Merge branch 'main' into task/restore_and_refactor_c_letter_detectors
amanfcp Jul 31, 2025
033ae77
Merge branch 'main' into task/restore_and_refactor_c_letter_detectors
kashifkhan0771 Aug 6, 2025
3e0403a
Merge branch 'main' into task/restore_and_refactor_c_letter_detectors
amanfcp Aug 7, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 31 additions & 12 deletions pkg/detectors/caflou/caflou.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package caflou
import (
"context"
"fmt"
"io"
"net/http"
"strings"
"time"
Expand Down Expand Up @@ -54,18 +55,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
client = defaultClient
}

req, err := http.NewRequestWithContext(ctx, "GET", "https://app.caflou.com/api/v1/accounts", nil)
if err != nil {
continue
}
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
}
}
isVerified, verificationErr := verifyMatch(ctx, client, resMatch)
s1.Verified = isVerified
s1.SetVerificationError(verificationErr, resMatch)
}

results = append(results, s1)
Expand All @@ -74,6 +66,33 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
return results, nil
}

func verifyMatch(ctx context.Context, client *http.Client, token string) (bool, error) {
req, err := http.NewRequestWithContext(ctx, "GET", "https://app.caflou.com/api/v1/accounts", nil)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: Seems like a good opportunity to replace nil and hard coded strings with http.NoBody and http.MethodGet in all of the detectors

if err != nil {
return false, err
}
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))
res, err := client.Do(req)

if err != nil {
return false, err
}

defer func() {
_, _ = io.Copy(io.Discard, res.Body)
_ = res.Body.Close()
}()

switch res.StatusCode {
case http.StatusOK:
return true, nil
case http.StatusUnauthorized:
return false, nil
default:
return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
}

func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Caflou
}
Expand Down
4 changes: 2 additions & 2 deletions pkg/detectors/caflou/caflou_integration_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,12 @@ import (
func TestCaflou_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors6")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("CAFLOU")
inactiveSecret := testSecrets.MustGetField("CAFLOU_INACTIVE")
inactiveSecret := testSecrets.MustGetField("CAFLOU_INVALID")

type args struct {
ctx context.Context
Expand Down
42 changes: 31 additions & 11 deletions pkg/detectors/calendarific/calendarific.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ package calendarific

import (
"context"
"fmt"
"io"
"net/http"
"strings"

Expand Down Expand Up @@ -45,17 +47,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}

if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://calendarific.com/api/v2/holidays?&api_key="+resMatch+"&country=US&year=2019", nil)
if err != nil {
continue
}
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
}
}
isVerified, verificationErr := verifyMatch(ctx, client, resMatch)
s1.Verified = isVerified
s1.SetVerificationError(verificationErr, resMatch)
}

results = append(results, s1)
Expand All @@ -64,6 +58,32 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
return results, nil
}

func verifyMatch(ctx context.Context, client *http.Client, token string) (bool, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://calendarific.com/api/v2/holidays?&api_key="+token+"&country=US&year=2019", http.NoBody)
if err != nil {
return false, err
}
res, err := client.Do(req)

if err != nil {
return false, err
}

defer func() {
_, _ = io.Copy(io.Discard, res.Body)
_ = res.Body.Close()
}()

switch res.StatusCode {
case http.StatusOK:
return true, nil
case http.StatusUnauthorized:
return false, nil
default:
return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
}

func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Calendarific
}
Expand Down
45 changes: 32 additions & 13 deletions pkg/detectors/calendlyapikey/calendlyapikey.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package calendlyapikey
import (
"context"
"fmt"
"io"
"net/http"
"strings"

Expand All @@ -22,7 +23,7 @@ var (
client = common.SaneHttpClient()

// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"calendly"}) + `\b(eyJ[A-Za-z0-9-_]{100,300}\.eyJ[A-Za-z0-9-_]{100,300}\.[A-Za-z0-9-_]+)\b`)
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"calendly"}) + `\b(eyJ[A-Za-z0-9-_]{10,300}\.eyJ[A-Za-z0-9-_]{10,300}\.[A-Za-z0-9-_]+)\b`)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In my opinion, this change could lead to more false positives. We're essentially reducing our safeguards. Are we confident about this change? Also, have we verified whether Calendly actually generates keys of this shorter length?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I made this change cos I saw some valid keys that were less than 100 letters. If we keep it as is, we're potentially increasing false negatives.

I'll confirm this again.

)

// Keywords are used for efficiently pre-filtering chunks.
Expand All @@ -46,18 +47,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}

if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.calendly.com/users/me", nil)
if err != nil {
continue
}
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
}
}
isVerified, verificationErr := verifyMatch(ctx, client, resMatch)
s1.Verified = isVerified
s1.SetVerificationError(verificationErr, resMatch)
}

results = append(results, s1)
Expand All @@ -66,6 +58,33 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
return results, nil
}

func verifyMatch(ctx context.Context, client *http.Client, token string) (bool, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://api.calendly.com/users/me", http.NoBody)
if err != nil {
return false, err
}
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))
res, err := client.Do(req)

if err != nil {
return false, err
}

defer func() {
_, _ = io.Copy(io.Discard, res.Body)
_ = res.Body.Close()
}()

switch res.StatusCode {
case http.StatusOK:
return true, nil
case http.StatusUnauthorized:
return false, nil
default:
return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
}

func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_CalendlyApiKey
}
Expand Down
49 changes: 35 additions & 14 deletions pkg/detectors/calorieninja/calorieninja.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ package calorieninja

import (
"context"
"fmt"
"io"
"net/http"
"strings"

Expand All @@ -21,7 +23,7 @@ var (
client = common.SaneHttpClient()

// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"calorieninja"}) + `\b([0-9A-Za-z]{40})\b`)
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"calorieninja"}) + `\b([0-9A-Za-z=+/]{40})\b`)
)

// Keywords are used for efficiently pre-filtering chunks.
Expand All @@ -45,19 +47,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}

if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.calorieninjas.com/v1/nutrition?query", nil)
if err != nil {
continue
}
req.Header.Add("Content-Type", "application/json")
req.Header.Add("X-Api-Key", resMatch)
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
}
}
isVerified, verificationErr := verifyMatch(ctx, client, resMatch)
s1.Verified = isVerified
s1.SetVerificationError(verificationErr, resMatch)
}

results = append(results, s1)
Expand All @@ -66,6 +58,35 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
return results, nil
}

func verifyMatch(ctx context.Context, client *http.Client, token string) (bool, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://api.calorieninjas.com/v1/nutrition?query", http.NoBody)
if err != nil {
return false, err
}
req.Header.Add("Content-Type", "application/json")
req.Header.Add("X-Api-Key", token)
res, err := client.Do(req)

if err != nil {
return false, err
}

defer func() {
_, _ = io.Copy(io.Discard, res.Body)
_ = res.Body.Close()
}()

switch res.StatusCode {
case http.StatusOK:
return true, nil
// Invalid API key returns 400 bad request
case http.StatusBadRequest:
return false, nil
default:
return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
}

func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_CalorieNinja
}
Expand Down
2 changes: 1 addition & 1 deletion pkg/detectors/calorieninja/calorieninja_integration_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ import (
func TestCalorieninja_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors6")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
Expand Down
10 changes: 10 additions & 0 deletions pkg/detectors/calorieninja/calorieninja_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,12 @@ var (
# - Remember to rotate the secret every 90 days.
# - The above credentials should only be used in a secure environment.
`
validPatternWithBase64 = `
# Configuration with base64-like API key
calorieninja_api_key: "e2xRY0yQmqhSetiQx0ZKWg==QMP1HstAoHdzP8qg"
`
secret = "ix1aaifujilTcGEjB67e1EBBRXcr7r9cdChAR5hb"
secretBase64 = "e2xRY0yQmqhSetiQx0ZKWg==QMP1HstAoHdzP8qg"
)

func TestCalorieNinja_Pattern(t *testing.T) {
Expand All @@ -45,6 +50,11 @@ func TestCalorieNinja_Pattern(t *testing.T) {
input: validPattern,
want: []string{secret},
},
{
name: "valid pattern with base64 characters",
input: validPatternWithBase64,
want: []string{secretBase64},
},
}

for _, test := range tests {
Expand Down
Loading
Loading