Add support for defining sources via a config file (#4172)

* Add config.proto

* Make protos

* Add ConfigurableSource implementation

* Parse config protoyaml into configured sources

* Refactor setting postman keywords into a helper method on the engine

* Add scan subcommand and plumb it all together

* Refactor ConfigurableSource to ConfiguredSource

* Export AhoCorasickCoreKeywords and return the set

* Disallow source configurations for non-scan subcommands

* Fix stdin scan compilation errors

* Rename command to multi-scan

* Add config file documentation

* Add more documentation for ConfiguredSource

* Replace initFunc closure with an embedded anonymous struct

* Assign directly to refs

* Fix typo

Author: mcastorina

README.md

@@ -418,6 +418,7 @@ TruffleHog has a sub-command for each source of data that you may want to scan:
 - jenkins
 - elasticsearch
 - stdin
+- multi-scan
 
 Each subcommand can have options that you can see with the `--help` flag provided to the sub command:
 
@@ -481,6 +482,33 @@ For example, to scan a `git` repository, start with
 trufflehog git https://github.com/trufflesecurity/trufflehog.git
 ```
 
+## Configuration
+
+TruffleHog supports defining [custom regex detectors](#regex-detector-alpha)
+and multiple sources in a configuration file provided via the `--config` flag.
+The regex detectors can be used with any subcommand, while the sources defined
+in configuration are only for the `multi-scan` subcommand.
+
+The configuration format for sources can be found on Truffle Security's
+[source configuration documentation page](https://docs.trufflesecurity.com/scan-data-for-secrets).
+
+Example GitHub source configuration and [options reference](https://docs.trufflesecurity.com/github#Fvm1I):
+
+```yaml
+sources:
+- connection:
+    '@type': type.googleapis.com/sources.GitHub
+    repositories:
+    - https://github.com/trufflesecurity/test_keys.git
+    unauthenticated: {}
+  name: example config scan
+  type: SOURCE_TYPE_GITHUB
+  verify: true
+```
+
+You may define multiple connections under the `sources` key (see above), and
+TruffleHog will scan all of the sources concurrently.
+
 ## S3
 
 The S3 source supports assuming IAM roles for scanning in addition to IAM users. This makes it easier for users to scan multiple AWS accounts without needing to rely on hardcoded credentials for each account.

main.go

@@ -255,6 +255,7 @@ var (
 	huggingfaceIncludePrs         = huggingfaceScan.Flag("include-prs", "Include pull requests in scan.").Bool()
 
 	stdinInputScan = cli.Command("stdin", "Find credentials from stdin.")
+	multiScanScan  = cli.Command("multi-scan", "Find credentials in multiple sources defined in configuration.")
 
 	analyzeCmd = analyzer.Command(cli)
 	usingTUI   = false
@@ -515,7 +516,8 @@ func run(state overseer.State) {
 	verificationCacheMetrics := verificationcache.InMemoryMetrics{}
 
 	engConf := engine.Config{
-		Concurrency: *concurrency,
+		Concurrency:       *concurrency,
+		ConfiguredSources: conf.Sources,
 		// The engine must always be configured with the list of
 		// default detectors, which can be further filtered by the
 		// user. The filters are applied by the engine and are only
@@ -540,6 +542,16 @@ func run(state overseer.State) {
 		engConf.VerificationResultCache = simple.NewCache[detectors.Result]()
 	}
 
+	// Check that there are no sources defined for non-scan subcommands. If
+	// there are, return an error as it is ambiguous what the user is
+	// trying to do.
+	if cmd != multiScanScan.FullCommand() && len(conf.Sources) > 0 {
+		logFatal(
+			fmt.Errorf("ambiguous configuration"),
+			"sources should only be defined in configuration for the 'multi-scan' command",
+		)
+	}
+
 	if *compareDetectionStrategies {
 		if err := compareScans(ctx, cmd, engConf); err != nil {
 			logFatal(err, "error comparing detection strategies")
@@ -702,7 +714,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
 		}
 	}()
 
-	var ref sources.JobProgressRef
+	var refs []sources.JobProgressRef
 	switch cmd {
 	case gitScan.FullCommand():
 		gitCfg := sources.GitConfig{
@@ -715,8 +727,10 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
 			Bare:             *gitScanBare,
 			ExcludeGlobs:     *gitScanExcludeGlobs,
 		}
-		if ref, err = eng.ScanGit(ctx, gitCfg); err != nil {
+		if ref, err := eng.ScanGit(ctx, gitCfg); err != nil {
 			return scanMetrics, fmt.Errorf("failed to scan Git: %v", err)
+		} else {
+			refs = []sources.JobProgressRef{ref}
 		}
 	case githubScan.FullCommand():
 		filter, err := common.FilterFromFiles(*githubScanIncludePaths, *githubScanExcludePaths)
@@ -745,8 +759,10 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
 			Filter:                     filter,
 			AuthInUrl:                  *githubAuthInUrl,
 		}
-		if ref, err = eng.ScanGitHub(ctx, cfg); err != nil {
+		if ref, err := eng.ScanGitHub(ctx, cfg); err != nil {
 			return scanMetrics, fmt.Errorf("failed to scan Github: %v", err)
+		} else {
+			refs = []sources.JobProgressRef{ref}
 		}
 	case githubExperimentalScan.FullCommand():
 		cfg := sources.GitHubExperimentalConfig{
@@ -756,8 +772,10 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
 			CollisionThreshold: *githubExperimentalCollisionThreshold,
 			DeleteCachedData:   *githubExperimentalDeleteCache,
 		}
-		if ref, err = eng.ScanGitHubExperimental(ctx, cfg); err != nil {
+		if ref, err := eng.ScanGitHubExperimental(ctx, cfg); err != nil {
 			return scanMetrics, fmt.Errorf("failed to scan using Github Experimental: %v", err)
+		} else {
+			refs = []sources.JobProgressRef{ref}
 		}
 	case gitlabScan.FullCommand():
 		filter, err := common.FilterFromFiles(*gitlabScanIncludePaths, *gitlabScanExcludePaths)
@@ -774,8 +792,10 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
 			Filter:       filter,
 			AuthInUrl:    *gitlabAuthInUrl,
 		}
-		if ref, err = eng.ScanGitLab(ctx, cfg); err != nil {
+		if ref, err := eng.ScanGitLab(ctx, cfg); err != nil {
 			return scanMetrics, fmt.Errorf("failed to scan GitLab: %v", err)
+		} else {
+			refs = []sources.JobProgressRef{ref}
 		}
 	case filesystemScan.FullCommand():
 		if len(*filesystemDirectories) > 0 {
@@ -789,8 +809,10 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
 			IncludePathsFile: *filesystemScanIncludePaths,
 			ExcludePathsFile: *filesystemScanExcludePaths,
 		}
-		if ref, err = eng.ScanFileSystem(ctx, cfg); err != nil {
+		if ref, err := eng.ScanFileSystem(ctx, cfg); err != nil {
 			return scanMetrics, fmt.Errorf("failed to scan filesystem: %v", err)
+		} else {
+			refs = []sources.JobProgressRef{ref}
 		}
 	case s3Scan.FullCommand():
 		cfg := sources.S3Config{
@@ -803,8 +825,10 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
 			CloudCred:     *s3ScanCloudEnv,
 			MaxObjectSize: int64(*s3ScanMaxObjectSize),
 		}
-		if ref, err = eng.ScanS3(ctx, cfg); err != nil {
+		if ref, err := eng.ScanS3(ctx, cfg); err != nil {
 			return scanMetrics, fmt.Errorf("failed to scan S3: %v", err)
+		} else {
+			refs = []sources.JobProgressRef{ref}
 		}
 	case syslogScan.FullCommand():
 		cfg := sources.SyslogConfig{
@@ -815,16 +839,22 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
 			KeyPath:     *syslogTLSKey,
 			Concurrency: *concurrency,
 		}
-		if ref, err = eng.ScanSyslog(ctx, cfg); err != nil {
+		if ref, err := eng.ScanSyslog(ctx, cfg); err != nil {
 			return scanMetrics, fmt.Errorf("failed to scan syslog: %v", err)
+		} else {
+			refs = []sources.JobProgressRef{ref}
 		}
 	case circleCiScan.FullCommand():
-		if ref, err = eng.ScanCircleCI(ctx, *circleCiScanToken); err != nil {
+		if ref, err := eng.ScanCircleCI(ctx, *circleCiScanToken); err != nil {
 			return scanMetrics, fmt.Errorf("failed to scan CircleCI: %v", err)
+		} else {
+			refs = []sources.JobProgressRef{ref}
 		}
 	case travisCiScan.FullCommand():
-		if ref, err = eng.ScanTravisCI(ctx, *travisCiScanToken); err != nil {
+		if ref, err := eng.ScanTravisCI(ctx, *travisCiScanToken); err != nil {
 			return scanMetrics, fmt.Errorf("failed to scan TravisCI: %v", err)
+		} else {
+			refs = []sources.JobProgressRef{ref}
 		}
 	case gcsScan.FullCommand():
 		cfg := sources.GCSConfig{
@@ -840,17 +870,21 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
 			Concurrency:    *concurrency,
 			MaxObjectSize:  int64(*gcsMaxObjectSize),
 		}
-		if ref, err = eng.ScanGCS(ctx, cfg); err != nil {
+		if ref, err := eng.ScanGCS(ctx, cfg); err != nil {
 			return scanMetrics, fmt.Errorf("failed to scan GCS: %v", err)
+		} else {
+			refs = []sources.JobProgressRef{ref}
 		}
 	case dockerScan.FullCommand():
 		cfg := sources.DockerConfig{
 			BearerToken:       *dockerScanToken,
 			Images:            *dockerScanImages,
 			UseDockerKeychain: *dockerScanToken == "",
 		}
-		if ref, err = eng.ScanDocker(ctx, cfg); err != nil {
+		if ref, err := eng.ScanDocker(ctx, cfg); err != nil {
 			return scanMetrics, fmt.Errorf("failed to scan Docker: %v", err)
+		} else {
+			refs = []sources.JobProgressRef{ref}
 		}
 	case postmanScan.FullCommand():
 		// handle deprecated flag
@@ -886,8 +920,10 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
 			WorkspacePaths:      *postmanWorkspacePaths,
 			EnvironmentPaths:    *postmanEnvironmentPaths,
 		}
-		if ref, err = eng.ScanPostman(ctx, cfg); err != nil {
+		if ref, err := eng.ScanPostman(ctx, cfg); err != nil {
 			return scanMetrics, fmt.Errorf("failed to scan Postman: %v", err)
+		} else {
+			refs = []sources.JobProgressRef{ref}
 		}
 	case elasticsearchScan.FullCommand():
 		cfg := sources.ElasticsearchConfig{
@@ -902,8 +938,10 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
 			SinceTimestamp: *elasticsearchSinceTimestamp,
 			BestEffortScan: *elasticsearchBestEffortScan,
 		}
-		if ref, err = eng.ScanElasticsearch(ctx, cfg); err != nil {
+		if ref, err := eng.ScanElasticsearch(ctx, cfg); err != nil {
 			return scanMetrics, fmt.Errorf("failed to scan Elasticsearch: %v", err)
+		} else {
+			refs = []sources.JobProgressRef{ref}
 		}
 	case jenkinsScan.FullCommand():
 		cfg := engine.JenkinsConfig{
@@ -912,8 +950,10 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
 			Username:              *jenkinsUsername,
 			Password:              *jenkinsPassword,
 		}
-		if ref, err = eng.ScanJenkins(ctx, cfg); err != nil {
+		if ref, err := eng.ScanJenkins(ctx, cfg); err != nil {
 			return scanMetrics, fmt.Errorf("failed to scan Jenkins: %v", err)
+		} else {
+			refs = []sources.JobProgressRef{ref}
 		}
 	case huggingfaceScan.FullCommand():
 		if *huggingfaceEndpoint != "" {
@@ -945,13 +985,26 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
 			IncludePrs:         *huggingfaceIncludePrs,
 			Concurrency:        *concurrency,
 		}
-		if ref, err = eng.ScanHuggingface(ctx, cfg); err != nil {
+		if ref, err := eng.ScanHuggingface(ctx, cfg); err != nil {
 			return scanMetrics, fmt.Errorf("failed to scan HuggingFace: %v", err)
+		} else {
+			refs = []sources.JobProgressRef{ref}
+		}
+	case multiScanScan.FullCommand():
+		if *configFilename == "" {
+			return scanMetrics, fmt.Errorf("missing required flag: --config")
+		}
+		if rs, err := eng.ScanConfig(ctx, cfg.ConfiguredSources...); err != nil {
+			return scanMetrics, fmt.Errorf("failed to scan via config: %w", err)
+		} else {
+			refs = rs
 		}
 	case stdinInputScan.FullCommand():
 		cfg := sources.StdinConfig{}
-		if ref, err = eng.ScanStdinInput(ctx, cfg); err != nil {
+		if ref, err := eng.ScanStdinInput(ctx, cfg); err != nil {
 			return scanMetrics, fmt.Errorf("failed to scan stdin input: %v", err)
+		} else {
+			refs = []sources.JobProgressRef{ref}
 		}
 	default:
 		return scanMetrics, fmt.Errorf("invalid command: %s", cmd)
@@ -962,13 +1015,19 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
 		return scanMetrics, fmt.Errorf("engine failed to finish execution: %v", err)
 	}
 
-	// Print any errors reported during the scan.
-	if errs := ref.Snapshot().Errors; len(errs) > 0 {
-		errMsgs := make([]string, len(errs))
-		for i := 0; i < len(errs); i++ {
-			errMsgs[i] = errs[i].Error()
+	// Print any non-fatal errors reported during the scan.
+	for _, ref := range refs {
+		if errs := ref.Snapshot().Errors; len(errs) > 0 {
+			errMsgs := make([]string, len(errs))
+			for i := 0; i < len(errs); i++ {
+				errMsgs[i] = errs[i].Error()
+			}
+			ctx.Logger().Error(nil, "encountered errors during scan",
+				"job", ref.JobID,
+				"source_name", ref.SourceName,
+				"errors", errMsgs,
+			)
 		}
-		ctx.Logger().Error(nil, "encountered errors during scan", "errors", errMsgs)
 	}
 
 	if *printAvgDetectorTime {