add ratelimit option #251

Author: edoardottt

README.md

@@ -71,6 +71,7 @@ CONFIGURATIONS:
    -hash string[]        Filter results having these favicon hashes (comma separated)
    -c, -concurrency int  Concurrency level (default 50)
    -t, -timeout int      Connection timeout in seconds (default 10)
+   -rl, -rate-limit int  Set a rate limit (per second)
 
 OUTPUT:
    -o, -output string  File to write output results

go.mod

@@ -11,6 +11,7 @@ require (
 
 require (
 	github.com/andybalholm/brotli v1.0.6 // indirect
+	github.com/benbjohnson/clock v1.3.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/klauspost/compress v1.16.7 // indirect
 	github.com/klauspost/pgzip v1.2.5 // indirect
@@ -45,6 +46,7 @@ require (
 	github.com/twmb/murmur3 v1.1.8
 	github.com/ulikunitz/xz v0.5.11 // indirect
 	github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
+	go.uber.org/ratelimit v0.3.0
 	golang.org/x/net v0.17.0 // indirect
 	gopkg.in/djherbis/times.v1 v1.3.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect

go.sum

@@ -5,6 +5,8 @@ github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3d
 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
 github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
 github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
+github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A=
+github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ=
 github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08/go.mod h1:pCxVEbcm3AMg7ejXyorUXi6HQCzOIBf7zEDVPtw0/U4=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -77,6 +79,8 @@ github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8=
 github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
 github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo=
 github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos=
+go.uber.org/ratelimit v0.3.0 h1:IdZd9wqvFXnvLvSEBo0KPcGfkoBGNkpTHlrE3Rcjkjw=
+go.uber.org/ratelimit v0.3.0/go.mod h1:So5LG7CV1zWpY1sHe+DXTJqQvOx+FFPFaAs2SnoyBaI=
 golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
 golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
 golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY=

pkg/favirecon/favirecon.go

@@ -129,6 +129,8 @@ func pushInput(r *Runner) {
 func execute(r *Runner) {
 	defer r.InWg.Done()
 
+	rl := rateLimiter(r)
+
 	for i := 0; i < r.Options.Concurrency; i++ {
 		r.InWg.Add(1)
 
@@ -145,6 +147,8 @@ func execute(r *Runner) {
 					return
 				}
 
+				rl.Take()
+
 				client := customClient(r.Options.Timeout)
 
 				result, err := getFavicon(targetURL, r.UserAgent, client)

pkg/favirecon/ratelimit.go

@@ -0,0 +1,20 @@
+/*
+favirecon - Use favicon.ico to improve your target recon phase. Quickly detect technologies, WAF, exposed panels, known services.
+
+This repository is under MIT License https://github.com/edoardottt/favirecon/blob/main/LICENSE
+*/
+
+package favirecon
+
+import "go.uber.org/ratelimit"
+
+func rateLimiter(r *Runner) ratelimit.Limiter {
+	var ratelimiter ratelimit.Limiter
+	if r.Options.RateLimit > 0 {
+		ratelimiter = ratelimit.New(r.Options.RateLimit)
+	} else {
+		ratelimiter = ratelimit.NewUnlimited()
+	}
+
+	return ratelimiter
+}

pkg/input/check.go

@@ -29,7 +29,11 @@ func (options *Options) validateOptions() error {
 	}
 
 	if options.Concurrency <= 0 {
-		return fmt.Errorf("%w", ErrNegativeValue)
+		return fmt.Errorf("concurrency: %w", ErrNegativeValue)
+	}
+
+	if options.RateLimit != 0 && options.RateLimit <= 0 {
+		return fmt.Errorf("rate limit: %w", ErrNegativeValue)
 	}
 
 	return nil

pkg/input/flags.go

@@ -20,6 +20,7 @@ import (
 const (
 	DefaultTimeout     = 10
 	DefaultConcurrency = 50
+	DefaultRateLimit   = 0
 )
 
 // Options struct specifies how the tool
@@ -35,6 +36,7 @@ type Options struct {
 	Concurrency int
 	Timeout     int
 	Cidr        bool
+	RateLimit   int
 }
 
 // configureOutput configures the output on the screen.
@@ -64,6 +66,7 @@ func ParseOptions() *Options {
 		flagSet.StringSliceVarP(&options.Hash, "hash", "", nil, `Filter results having these favicon hashes (comma separated)`, goflags.CommaSeparatedStringSliceOptions),
 		flagSet.IntVarP(&options.Concurrency, "concurrency", "c", DefaultConcurrency, `Concurrency level`),
 		flagSet.IntVarP(&options.Timeout, "timeout", "t", DefaultTimeout, `Connection timeout in seconds`),
+		flagSet.IntVarP(&options.RateLimit, "rate-limit", "rl", DefaultRateLimit, `Set a rate limit (per second)`),
 	)
 
 	// Output