Skip to content

[INS-202] Add rate limiting to the Github Analyzer #4617

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mustansir14 merged 5 commits into from
Jan 19, 2026
Merged

[INS-202] Add rate limiting to the Github Analyzer #4617

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
a559220
add analyzer client option to support rate limiting. Add rate limitin…
mustansir14 Dec 24, 2025
c5005e4
supply the rate limiter to the client
mustansir14 Dec 24, 2025
5b7da9e
Merge branch 'main' into INS-202-Gracefully-Handle-Rate-Limits-In-Git…
mustansir14 Jan 12, 2026
956f9eb
Merge branch 'main' into INS-202-Gracefully-Handle-Rate-Limits-In-Git…
mustansir14 Jan 14, 2026
c5ac682
Merge branch 'main' into INS-202-Gracefully-Handle-Rate-Limits-In-Git…
mustansir14 Jan 19, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 51 additions & 16 deletions pkg/analyzer/analyzers/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"time"

"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/config"
"golang.org/x/time/rate"
)

type AnalyzeClient struct {
Expand All @@ -28,36 +29,44 @@ func CreateLogFileName(baseName string) string {
return logFileName
}

type ClientOption func(*http.Client)

// This returns a client that is restricted and filters out unsafe requests returning a success status code.
func NewAnalyzeClient(cfg *config.Config) *http.Client {
func NewAnalyzeClient(cfg *config.Config, opts ...func(*http.Client)) *http.Client {
client := &http.Client{
Transport: AnalyzerRoundTripper{parent: http.DefaultTransport},
}
if cfg == nil || !cfg.LoggingEnabled {
return client
if cfg != nil && cfg.LoggingEnabled {
client = &http.Client{
Transport: LoggingRoundTripper{
parent: client.Transport,
logFile: cfg.LogFile,
},
}
}
return &http.Client{
Transport: LoggingRoundTripper{
parent: client.Transport,
logFile: cfg.LogFile,
},
for _, opt := range opts {
opt(client)
}
return client
}

// This returns a client that is unrestricted and does not filter out unsafe requests returning a success status code.
func NewAnalyzeClientUnrestricted(cfg *config.Config) *http.Client {
func NewAnalyzeClientUnrestricted(cfg *config.Config, opts ...ClientOption) *http.Client {
client := &http.Client{
Transport: http.DefaultTransport,
}
if cfg == nil || !cfg.LoggingEnabled {
return client
if cfg != nil && cfg.LoggingEnabled {
client = &http.Client{
Transport: LoggingRoundTripper{
parent: client.Transport,
logFile: cfg.LogFile,
},
}
}
return &http.Client{
Transport: LoggingRoundTripper{
parent: client.Transport,
logFile: cfg.LogFile,
},
for _, opt := range opts {
opt(client)
}
return client
}

type LoggingRoundTripper struct {
Expand Down Expand Up @@ -134,3 +143,29 @@ func IsMethodSafe(method string) bool {
return false
}
}

type RateLimitRoundTripper struct {
parent http.RoundTripper
limiter *rate.Limiter
}

func (rt RateLimitRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
if rt.parent == nil {
rt.parent = http.DefaultTransport
}
if rt.limiter != nil {
if err := rt.limiter.Wait(req.Context()); err != nil {
return nil, err
}
}
return rt.parent.RoundTrip(req)
}

func WithRateLimiter(l *rate.Limiter) ClientOption {
return func(c *http.Client) {
c.Transport = RateLimitRoundTripper{
parent: c.Transport,
limiter: l,
}
}
}
10 changes: 9 additions & 1 deletion pkg/analyzer/analyzers/github/github.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (

"github.com/fatih/color"
gh "github.com/google/go-github/v67/github"
"golang.org/x/time/rate"

"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/analyzers"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/analyzers/github/classic"
Expand All @@ -16,6 +17,13 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
)

// According to GitHub's rate limiting documentation, the default rate limit for
// authenticated requests (PAT) is 5000 requests per hour. This equates to roughly 1.39
// requests per second. To provide some buffer, we set the rate limit to 1.25
// requests per second with a burst of 10.
// https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api?apiVersion=2022-11-28
var rateLimiter = rate.NewLimiter(rate.Limit(1.25), 10)

var _ analyzers.Analyzer = (*Analyzer)(nil)

type Analyzer struct {
Expand Down Expand Up @@ -132,7 +140,7 @@ func AnalyzePermissions(cfg *config.Config, key string) (*common.SecretInfo, err
if cfg == nil {
cfg = &config.Config{}
}
client := gh.NewClient(analyzers.NewAnalyzeClient(cfg)).WithAuthToken(key)
client := gh.NewClient(analyzers.NewAnalyzeClient(cfg, analyzers.WithRateLimiter(rateLimiter))).WithAuthToken(key)

md, err := common.GetTokenMetadata(key, client)
if err != nil {
Expand Down
2 changes: 1 addition & 1 deletion pkg/analyzer/analyzers/github/github_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,7 @@ func TestAnalyzer_Analyze(t *testing.T) {
"UnboundedResources": null,
"Metadata": {
"owner": "sirdetectsalot",
"expiration": "2025-08-05T00:00:00-07:00",
"expiration": "2026-03-24T15:27:38+05:00",
"type": "Fine-Grained GitHub Personal Access Token"
}
}`,
Expand Down
Loading