[bug] - Ensure detector HTTP clients share the same timeout set at runtime (#3946)

Author: ahrav

main.go

@@ -20,16 +20,15 @@ import (
 	"github.com/go-logr/logr"
 	"github.com/jpillora/overseer"
 	"github.com/mattn/go-isatty"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/cache/simple"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/verificationcache"
 	"go.uber.org/automaxprocs/maxprocs"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/cache/simple"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/cleantemp"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/config"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/engine"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/defaults"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/feature"
@@ -39,6 +38,7 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/tui"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/updater"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/verificationcache"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/version"
 )
 
@@ -446,7 +446,9 @@ func run(state overseer.State) {
 	}
 
 	if *detectorTimeout != 0 {
+		logger.Info("Setting detector timeout", "timeout", detectorTimeout.String())
 		engine.SetDetectorTimeout(*detectorTimeout)
+		detectors.OverrideDetectorTimeout(*detectorTimeout)
 	}
 	if *archiveMaxSize != 0 {
 		handlers.SetArchiveMaxSize(int(*archiveMaxSize))

pkg/detectors/http.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"net"
 	"net/http"
+	"sync"
 	"time"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/feature"
@@ -13,7 +14,8 @@ import (
 var DetectorHttpClientWithNoLocalAddresses *http.Client
 var DetectorHttpClientWithLocalAddresses *http.Client
 
-const DefaultResponseTimeout = 5 * time.Second
+// DefaultResponseTimeout is the default timeout for HTTP requests.
+const DefaultResponseTimeout = 10 * time.Second
 
 func userAgent() string {
 	if len(feature.UserAgentSuffix.Load()) > 0 {
@@ -36,6 +38,20 @@ func init() {
 	)
 }
 
+var overrideOnce sync.Once
+
+// OverrideDetectorTimeout overrides the default timeout for the detector HTTP clients.
+// It is guaranteed to only run once, subsequent calls will have no effect.
+// This should be called before any scans are started.
+func OverrideDetectorTimeout(timeout time.Duration) {
+    overrideOnce.Do(func() {
+        DetectorHttpClientWithLocalAddresses.Timeout = timeout
+        DetectorHttpClientWithNoLocalAddresses.Timeout = timeout
+    })
+}
+
+
+
 // ClientOption defines a function type that modifies an http.Client.
 type ClientOption func(*http.Client)
 

pkg/engine/engine.go

@@ -13,7 +13,6 @@ import (
 	"github.com/adrg/strutil"
 	"github.com/adrg/strutil/metrics"
 	lru "github.com/hashicorp/golang-lru/v2"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/verificationcache"
 	"google.golang.org/protobuf/proto"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
@@ -29,9 +28,10 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/verificationcache"
 )
 
-var detectionTimeout = 10 * time.Second
+var detectionTimeout = detectors.DefaultResponseTimeout
 
 var errOverlap = errors.New(
 	"More than one detector has found this result. For your safety, verification has been disabled." +