From a55922088c912f519a12b1d2b69a779d914cc672 Mon Sep 17 00:00:00 2001 From: Mustansir Muzaffar Date: Wed, 24 Dec 2025 15:54:48 +0500 Subject: [PATCH 1/2] add analyzer client option to support rate limiting. Add rate limiting to github analyzer --- pkg/analyzer/analyzers/client.go | 67 +++++++++++++++----- pkg/analyzer/analyzers/github/github.go | 8 +++ pkg/analyzer/analyzers/github/github_test.go | 2 +- 3 files changed, 60 insertions(+), 17 deletions(-) diff --git a/pkg/analyzer/analyzers/client.go b/pkg/analyzer/analyzers/client.go index 0c80a3af90ad..1bd9818676c8 100644 --- a/pkg/analyzer/analyzers/client.go +++ b/pkg/analyzer/analyzers/client.go @@ -8,6 +8,7 @@ import ( "time" "github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/config" + "golang.org/x/time/rate" ) type AnalyzeClient struct { @@ -28,36 +29,44 @@ func CreateLogFileName(baseName string) string { return logFileName } +type ClientOption func(*http.Client) + // This returns a client that is restricted and filters out unsafe requests returning a success status code. -func NewAnalyzeClient(cfg *config.Config) *http.Client { +func NewAnalyzeClient(cfg *config.Config, opts ...func(*http.Client)) *http.Client { client := &http.Client{ Transport: AnalyzerRoundTripper{parent: http.DefaultTransport}, } - if cfg == nil || !cfg.LoggingEnabled { - return client + if cfg != nil && cfg.LoggingEnabled { + client = &http.Client{ + Transport: LoggingRoundTripper{ + parent: client.Transport, + logFile: cfg.LogFile, + }, + } } - return &http.Client{ - Transport: LoggingRoundTripper{ - parent: client.Transport, - logFile: cfg.LogFile, - }, + for _, opt := range opts { + opt(client) } + return client } // This returns a client that is unrestricted and does not filter out unsafe requests returning a success status code. -func NewAnalyzeClientUnrestricted(cfg *config.Config) *http.Client { +func NewAnalyzeClientUnrestricted(cfg *config.Config, opts ...ClientOption) *http.Client { client := &http.Client{ Transport: http.DefaultTransport, } - if cfg == nil || !cfg.LoggingEnabled { - return client + if cfg != nil && cfg.LoggingEnabled { + client = &http.Client{ + Transport: LoggingRoundTripper{ + parent: client.Transport, + logFile: cfg.LogFile, + }, + } } - return &http.Client{ - Transport: LoggingRoundTripper{ - parent: client.Transport, - logFile: cfg.LogFile, - }, + for _, opt := range opts { + opt(client) } + return client } type LoggingRoundTripper struct { @@ -134,3 +143,29 @@ func IsMethodSafe(method string) bool { return false } } + +type RateLimitRoundTripper struct { + parent http.RoundTripper + limiter *rate.Limiter +} + +func (rt RateLimitRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + if rt.parent == nil { + rt.parent = http.DefaultTransport + } + if rt.limiter != nil { + if err := rt.limiter.Wait(req.Context()); err != nil { + return nil, err + } + } + return rt.parent.RoundTrip(req) +} + +func WithRateLimiter(l *rate.Limiter) ClientOption { + return func(c *http.Client) { + c.Transport = RateLimitRoundTripper{ + parent: c.Transport, + limiter: l, + } + } +} diff --git a/pkg/analyzer/analyzers/github/github.go b/pkg/analyzer/analyzers/github/github.go index 9cd25719345b..94ffa4e07cef 100644 --- a/pkg/analyzer/analyzers/github/github.go +++ b/pkg/analyzer/analyzers/github/github.go @@ -7,6 +7,7 @@ import ( "github.com/fatih/color" gh "github.com/google/go-github/v67/github" + "golang.org/x/time/rate" "github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/analyzers" "github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/analyzers/github/classic" @@ -16,6 +17,13 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/context" ) +// According to GitHub's rate limiting documentation, the default rate limit for +// authenticated requests (PAT) is 5000 requests per hour. This equates to roughly 1.39 +// requests per second. To provide some buffer, we set the rate limit to 1.25 +// requests per second with a burst of 10. +// https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api?apiVersion=2022-11-28 +var rateLimiter = rate.NewLimiter(rate.Limit(1.25), 10) + var _ analyzers.Analyzer = (*Analyzer)(nil) type Analyzer struct { diff --git a/pkg/analyzer/analyzers/github/github_test.go b/pkg/analyzer/analyzers/github/github_test.go index caa1e2ed836a..a3b9a9ee2e90 100644 --- a/pkg/analyzer/analyzers/github/github_test.go +++ b/pkg/analyzer/analyzers/github/github_test.go @@ -231,7 +231,7 @@ func TestAnalyzer_Analyze(t *testing.T) { "UnboundedResources": null, "Metadata": { "owner": "sirdetectsalot", - "expiration": "2025-08-05T00:00:00-07:00", + "expiration": "2026-03-24T15:27:38+05:00", "type": "Fine-Grained GitHub Personal Access Token" } }`, From c5005e47e27bdbe98d73e1398ddb9b0dcdc726ff Mon Sep 17 00:00:00 2001 From: Mustansir Muzaffar Date: Wed, 24 Dec 2025 16:15:52 +0500 Subject: [PATCH 2/2] supply the rate limiter to the client --- pkg/analyzer/analyzers/github/github.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/analyzer/analyzers/github/github.go b/pkg/analyzer/analyzers/github/github.go index 94ffa4e07cef..46124bee7cf5 100644 --- a/pkg/analyzer/analyzers/github/github.go +++ b/pkg/analyzer/analyzers/github/github.go @@ -140,7 +140,7 @@ func AnalyzePermissions(cfg *config.Config, key string) (*common.SecretInfo, err if cfg == nil { cfg = &config.Config{} } - client := gh.NewClient(analyzers.NewAnalyzeClient(cfg)).WithAuthToken(key) + client := gh.NewClient(analyzers.NewAnalyzeClient(cfg, analyzers.WithRateLimiter(rateLimiter))).WithAuthToken(key) md, err := common.GetTokenMetadata(key, client) if err != nil {