feat: add new -workers flag to configure number of workers during scanning

Author: wakeful

Readme.md

@@ -25,6 +25,8 @@ Usage spark:
     verbose log output
   -version
     show version
+  -workers int
+    number of workers used for scanning (default 2)
 
 
 $ spark -list-scanners

app.go

@@ -9,19 +9,26 @@ import (
 	"log/slog"
 	"strings"
 
+	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/config"
 	"github.com/aws/aws-sdk-go-v2/service/sts"
 	"golang.org/x/sync/errgroup"
 )
 
 // App represents a struct that provides functionality for interacting with the AWS services.
 type App struct {
-	accountID string
-	runners   []runner
-	stsClient stsClient
+	accountID   string
+	runners     []runner
+	stsClient   stsClient
+	workerLimit int
 }
 
-func newApp(ctx context.Context, check []runnerType, regions []string) (*App, error) {
+func newApp(
+	ctx context.Context,
+	check []runnerType,
+	regions []string,
+	workerLimit int,
+) (*App, error) {
 	if len(regions) == 0 {
 		return nil, errEmptyRegion
 	}
@@ -32,6 +39,10 @@ func newApp(ctx context.Context, check []runnerType, regions []string) (*App, er
 
 	regions = uniqRegions(regions)
 
+	if workerLimit < 1 {
+		workerLimit = 1
+	}
+
 	baseCfg, err := config.LoadDefaultConfig(ctx)
 	if err != nil {
 		return nil, fmt.Errorf("failed to load aws config, %w", err)
@@ -40,6 +51,18 @@ func newApp(ctx context.Context, check []runnerType, regions []string) (*App, er
 	stsCfg := baseCfg.Copy()
 	stsCfg.Region = regions[0]
 
+	runners := setUpRunners(baseCfg, check, regions)
+
+	return &App{
+		accountID:   "",
+		runners:     runners,
+		stsClient:   sts.NewFromConfig(stsCfg),
+		workerLimit: workerLimit,
+	}, nil
+}
+
+// setUpRunners initializes and returns a list of runners based on the specified configuration, checks, and regions.
+func setUpRunners(baseCfg aws.Config, check []runnerType, regions []string) []runner {
 	runners := make([]runner, 0)
 
 	for _, region := range regions {
@@ -64,18 +87,14 @@ func newApp(ctx context.Context, check []runnerType, regions []string) (*App, er
 		}
 	}
 
-	return &App{
-		accountID: "",
-		runners:   runners,
-		stsClient: sts.NewFromConfig(stsCfg),
-	}, nil
+	return runners
 }
 
 // Run executes the scan process on the target using all available runners,
 // and returns the collected results or an error.
 func (a *App) Run(ctx context.Context, target string) ([]Result, error) {
 	group, gCtx := errgroup.WithContext(ctx)
-	group.SetLimit(1)
+	group.SetLimit(a.workerLimit)
 
 	if strings.EqualFold(target, "self") {
 		slog.Debug("replacing self with account ID",

app_test.go

@@ -18,6 +18,7 @@ func Test_newApp(t *testing.T) {
 		name        string
 		check       []runnerType
 		regions     []string
+		workerLimit int
 		wantApp     bool
 		wantErr     bool
 		wantWorkers []runnerType
@@ -83,7 +84,7 @@ func Test_newApp(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			t.Parallel()
 
-			got, err := newApp(t.Context(), tt.check, tt.regions)
+			got, err := newApp(t.Context(), tt.check, tt.regions, tt.workerLimit)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("newApp() error = %v, wantErr %v", err, tt.wantErr)
 
@@ -164,7 +165,8 @@ func TestApp_Run(t *testing.T) {
 			t.Parallel()
 
 			a := &App{
-				runners: tt.runners,
+				runners:     tt.runners,
+				workerLimit: 1,
 			}
 
 			got, err := a.Run(tt.ctx, tt.target)

main.go

@@ -16,13 +16,16 @@ import (
 var version = "dev"
 
 func main() { //nolint:cyclop
+	const numberOfWorkers = 2
+
 	var (
 		target         = flag.String("target", "self", "target AWS account ID")
 		listScanners   = flag.Bool("list-scanners", false, "list available resource types")
 		showVersion    = flag.Bool("version", false, "show version")
 		verbose        = flag.Bool("verbose", false, "verbose log output")
 		scanAllRegions = flag.Bool("region-all", false, "scan all regions")
 		scannersAll    = flag.Bool("scan-all", false, "scan all resource types")
+		workerCount    = flag.Int("workers", numberOfWorkers, "number of workers used for scanning")
 		regionVars     StringSlice
 		scannersVars   StringSlice
 	)
@@ -92,6 +95,7 @@ func main() { //nolint:cyclop
 		ctx,
 		getRunners(scannersVars),
 		regionVars,
+		*workerCount,
 	)
 	if err != nil {
 		slog.Error("failed to initialize app", slog.String("error", err.Error()))