refact pkg/acquisition: extract loop method (#3615)

* refact pkg/acquisition: extract loop method

* lint

Author: mmetc

cmd/crowdsec/main.go

@@ -120,7 +120,7 @@ func LoadAcquisition(cConfig *csconfig.Config) ([]acquisition.DataSource, error)
 			return nil, fmt.Errorf("failed to configure datasource for %s: %w", flags.OneShotDSN, err)
 		}
 	} else {
-		dataSources, err = acquisition.LoadAcquisitionFromFile(cConfig.Crowdsec, cConfig.Prometheus)
+		dataSources, err = acquisition.LoadAcquisitionFromFiles(cConfig.Crowdsec, cConfig.Prometheus)
 		if err != nil {
 			return nil, err
 		}

pkg/acquisition/acquisition.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"maps"
 	"os"
 	"strings"
 
@@ -24,7 +25,6 @@ import (
 	"github.com/crowdsecurity/crowdsec/pkg/cwversion/component"
 	"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
 	"github.com/crowdsecurity/crowdsec/pkg/types"
-	"maps"
 )
 
 type DataSourceUnavailableError struct {
@@ -42,18 +42,18 @@ func (e *DataSourceUnavailableError) Unwrap() error {
 
 // The interface each datasource must implement
 type DataSource interface {
-	GetMetrics() []prometheus.Collector                                       // Returns pointers to metrics that are managed by the module
-	GetAggregMetrics() []prometheus.Collector                                 // Returns pointers to metrics that are managed by the module (aggregated mode, limits cardinality)
-	UnmarshalConfig(yamlConfig []byte) error                                             // Decode and pre-validate the YAML datasource - anything that can be checked before runtime
-	Configure(yamlConfig []byte, logger *log.Entry, metricsLevel int) error                                  // Complete the YAML datasource configuration and perform runtime checks.
-	ConfigureByDSN(dsn string, labels map[string]string, logger *log.Entry, uniqueID string) error       // Configure the datasource
-	GetMode() string                                                          // Get the mode (TAIL, CAT or SERVER)
-	GetName() string                                                          // Get the name of the module
-	OneShotAcquisition(ctx context.Context, out chan types.Event, acquisTomb *tomb.Tomb) error   // Start one shot acquisition(eg, cat a file)
-	StreamingAcquisition(ctx context.Context, out chan types.Event, acquisTomb *tomb.Tomb) error // Start live acquisition (eg, tail a file)
-	CanRun() error                                                            // Whether the datasource can run or not (eg, journalctl on BSD is a non-sense)
-	GetUuid() string                                                          // Get the unique identifier of the datasource
-	Dump() interface{}
+	GetMetrics() []prometheus.Collector                                                            // Returns pointers to metrics that are managed by the module
+	GetAggregMetrics() []prometheus.Collector                                                      // Returns pointers to metrics that are managed by the module (aggregated mode, limits cardinality)
+	UnmarshalConfig(yamlConfig []byte) error                                                       // Decode and pre-validate the YAML datasource - anything that can be checked before runtime
+	Configure(yamlConfig []byte, logger *log.Entry, metricsLevel int) error                        // Complete the YAML datasource configuration and perform runtime checks.
+	ConfigureByDSN(dsn string, labels map[string]string, logger *log.Entry, uniqueID string) error // Configure the datasource
+	GetMode() string                                                                               // Get the mode (TAIL, CAT or SERVER)
+	GetName() string                                                                               // Get the name of the module
+	OneShotAcquisition(ctx context.Context, out chan types.Event, acquisTomb *tomb.Tomb) error     // Start one shot acquisition(eg, cat a file)
+	StreamingAcquisition(ctx context.Context, out chan types.Event, acquisTomb *tomb.Tomb) error   // Start live acquisition (eg, tail a file)
+	CanRun() error                                                                                 // Whether the datasource can run or not (eg, journalctl on BSD is a non-sense)
+	GetUuid() string                                                                               // Get the unique identifier of the datasource
+	Dump() any
 }
 
 var (
@@ -180,7 +180,7 @@ func LoadAcquisitionFromDSN(dsn string, labels map[string]string, transformExpr
 	uniqueId := uuid.NewString()
 
 	if transformExpr != "" {
-		vm, err := expr.Compile(transformExpr, exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...)
+		vm, err := expr.Compile(transformExpr, exprhelpers.GetExprOptions(map[string]any{"evt": &types.Event{}})...)
 		if err != nil {
 			return nil, fmt.Errorf("while compiling transform expression '%s': %w", transformExpr, err)
 		}
@@ -216,113 +216,127 @@ func GetMetricsLevelFromPromCfg(prom *csconfig.PrometheusCfg) int {
 	return configuration.METRICS_FULL
 }
 
-// LoadAcquisitionFromFile unmarshals the configuration item and checks its availability
-func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg, prom *csconfig.PrometheusCfg) ([]DataSource, error) {
+// sourcesFromFile reads and parses one acquisition file into DataSources.
+func sourcesFromFile(acquisFile string, metrics_level int) ([]DataSource, error) {
 	var sources []DataSource
 
-	metrics_level := GetMetricsLevelFromPromCfg(prom)
-
-	for _, acquisFile := range config.AcquisitionFiles {
-		log.Infof("loading acquisition file : %s", acquisFile)
-
-		yamlFile, err := os.Open(acquisFile)
-		if err != nil {
-			return nil, err
-		}
-
-		defer yamlFile.Close()
+	log.Infof("loading acquisition file : %s", acquisFile)
 
-		acquisContent, err := io.ReadAll(yamlFile)
-		if err != nil {
-			return nil, fmt.Errorf("failed to read %s: %w", acquisFile, err)
-		}
+	yamlFile, err := os.Open(acquisFile)
+	if err != nil {
+		return nil, err
+	}
 
-		expandedAcquis := csstring.StrictExpand(string(acquisContent), os.LookupEnv)
+	defer yamlFile.Close()
 
-		dec := yaml.NewDecoder(strings.NewReader(expandedAcquis))
-		dec.SetStrict(true)
+	acquisContent, err := io.ReadAll(yamlFile)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read %s: %w", acquisFile, err)
+	}
 
-		idx := -1
+	expandedAcquis := csstring.StrictExpand(string(acquisContent), os.LookupEnv)
 
-		for {
-			var sub configuration.DataSourceCommonCfg
+	dec := yaml.NewDecoder(strings.NewReader(expandedAcquis))
+	dec.SetStrict(true)
 
-			idx += 1
+	idx := -1
 
-			err = dec.Decode(&sub)
-			if err != nil {
-				if !errors.Is(err, io.EOF) {
-					return nil, fmt.Errorf("failed to parse %s: %w", acquisFile, err)
-				}
+	for {
+		var sub configuration.DataSourceCommonCfg
 
-				log.Tracef("End of yaml file")
+		idx += 1
 
-				break
+		err = dec.Decode(&sub)
+		if err != nil {
+			if !errors.Is(err, io.EOF) {
+				return nil, fmt.Errorf("failed to parse %s: %w", acquisFile, err)
 			}
 
-			// for backward compat ('type' was not mandatory, detect it)
-			if guessType := detectBackwardCompatAcquis(sub); guessType != "" {
-				log.Debugf("datasource type missing in %s (position %d): detected 'source=%s'", acquisFile, idx, guessType)
+			log.Tracef("End of yaml file")
 
-				if sub.Source != "" && sub.Source != guessType {
-					log.Warnf("datasource type mismatch in %s (position %d): found '%s' but should probably be '%s'", acquisFile, idx, sub.Source, guessType)
-				}
+			break
+		}
 
-				sub.Source = guessType
-			}
-			// it's an empty item, skip it
-			if len(sub.Labels) == 0 {
-				if sub.Source == "" {
-					log.Debugf("skipping empty item in %s", acquisFile)
-					continue
-				}
+		// for backward compat ('type' was not mandatory, detect it)
+		if guessType := detectBackwardCompatAcquis(sub); guessType != "" {
+			log.Debugf("datasource type missing in %s (position %d): detected 'source=%s'", acquisFile, idx, guessType)
 
-				if sub.Source != "docker" {
-					// docker is the only source that can be empty
-					return nil, fmt.Errorf("missing labels in %s (position %d)", acquisFile, idx)
-				}
+			if sub.Source != "" && sub.Source != guessType {
+				log.Warnf("datasource type mismatch in %s (position %d): found '%s' but should probably be '%s'", acquisFile, idx, sub.Source, guessType)
 			}
 
+			sub.Source = guessType
+		}
+		// it's an empty item, skip it
+		if len(sub.Labels) == 0 {
 			if sub.Source == "" {
-				return nil, fmt.Errorf("data source type is empty ('source') in %s (position %d)", acquisFile, idx)
+				log.Debugf("skipping empty item in %s", acquisFile)
+				continue
 			}
 
-			// pre-check that the source is valid
-			_, err := GetDataSourceIface(sub.Source)
-			if err != nil {
-				return nil, fmt.Errorf("in file %s (position %d) - %w", acquisFile, idx, err)
+			if sub.Source != "docker" {
+				// docker is the only source that can be empty
+				return nil, fmt.Errorf("missing labels in %s (position %d)", acquisFile, idx)
 			}
+		}
 
-			uniqueId := uuid.NewString()
-			sub.UniqueId = uniqueId
+		if sub.Source == "" {
+			return nil, fmt.Errorf("data source type is empty ('source') in %s (position %d)", acquisFile, idx)
+		}
 
-			src, err := DataSourceConfigure(sub, metrics_level)
-			if err != nil {
-				var dserr *DataSourceUnavailableError
-				if errors.As(err, &dserr) {
-					log.Error(err)
-					continue
-				}
+		// pre-check that the source is valid
+		_, err := GetDataSourceIface(sub.Source)
+		if err != nil {
+			return nil, fmt.Errorf("in file %s (position %d) - %w", acquisFile, idx, err)
+		}
 
-				return nil, fmt.Errorf("while configuring datasource of type %s from %s (position %d): %w", sub.Source, acquisFile, idx, err)
+		uniqueId := uuid.NewString()
+		sub.UniqueId = uniqueId
+
+		src, err := DataSourceConfigure(sub, metrics_level)
+		if err != nil {
+			var dserr *DataSourceUnavailableError
+			if errors.As(err, &dserr) {
+				log.Error(err)
+				continue
 			}
 
-			if sub.TransformExpr != "" {
-				vm, err := expr.Compile(sub.TransformExpr, exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...)
-				if err != nil {
-					return nil, fmt.Errorf("while compiling transform expression '%s' for datasource %s in %s (position %d): %w", sub.TransformExpr, sub.Source, acquisFile, idx, err)
-				}
+			return nil, fmt.Errorf("while configuring datasource of type %s from %s (position %d): %w", sub.Source, acquisFile, idx, err)
+		}
 
-				transformRuntimes[uniqueId] = vm
+		if sub.TransformExpr != "" {
+			vm, err := expr.Compile(sub.TransformExpr, exprhelpers.GetExprOptions(map[string]any{"evt": &types.Event{}})...)
+			if err != nil {
+				return nil, fmt.Errorf("while compiling transform expression '%s' for datasource %s in %s (position %d): %w", sub.TransformExpr, sub.Source, acquisFile, idx, err)
 			}
 
-			sources = append(sources, src)
+			transformRuntimes[uniqueId] = vm
 		}
+
+		sources = append(sources, src)
 	}
 
 	return sources, nil
 }
 
+// LoadAcquisitionFromFiles unmarshals the configuration item and checks its availability
+func LoadAcquisitionFromFiles(config *csconfig.CrowdsecServiceCfg, prom *csconfig.PrometheusCfg) ([]DataSource, error) {
+	var allSources []DataSource
+
+	metrics_level := GetMetricsLevelFromPromCfg(prom)
+
+	for _, acquisFile := range config.AcquisitionFiles {
+		sources, err := sourcesFromFile(acquisFile, metrics_level)
+		if err != nil {
+			return nil, err
+		}
+
+		allSources = append(allSources, sources...)
+	}
+
+	return allSources, nil
+}
+
 func GetMetrics(sources []DataSource, aggregated bool) error {
 	var metrics []prometheus.Collector
 
@@ -362,6 +376,7 @@ func copyEvent(evt types.Event, line string) types.Event {
 
 func transform(transformChan chan types.Event, output chan types.Event, acquisTomb *tomb.Tomb, transformRuntime *vm.Program, logger *log.Entry) {
 	defer trace.CatchPanic("crowdsec/acquis")
+
 	logger.Infof("transformer started")
 
 	for {
@@ -372,7 +387,7 @@ func transform(transformChan chan types.Event, output chan types.Event, acquisTo
 		case evt := <-transformChan:
 			logger.Tracef("Received event %s", evt.Line.Raw)
 
-			out, err := expr.Run(transformRuntime, map[string]interface{}{"evt": &evt})
+			out, err := expr.Run(transformRuntime, map[string]any{"evt": &evt})
 			if err != nil {
 				logger.Errorf("while running transform expression: %s, sending event as-is", err)
 				output <- evt

pkg/acquisition/acquisition_test.go

@@ -24,8 +24,9 @@ import (
 
 type MockSource struct {
 	configuration.DataSourceCommonCfg `yaml:",inline"`
-	Toto                              string `yaml:"toto"`
-	logger                            *log.Entry
+
+	Toto   string `yaml:"toto"`
+	logger *log.Entry
 }
 
 func (f *MockSource) UnmarshalConfig(cfg []byte) error {
@@ -215,7 +216,7 @@ wowo: ajsajasjas
 	}
 }
 
-func TestLoadAcquisitionFromFile(t *testing.T) {
+func TestLoadAcquisitionFromFiles(t *testing.T) {
 	appendMockSource()
 	t.Setenv("TEST_ENV", "test_value2")
 
@@ -293,7 +294,7 @@ func TestLoadAcquisitionFromFile(t *testing.T) {
 	}
 	for _, tc := range tests {
 		t.Run(tc.TestName, func(t *testing.T) {
-			dss, err := LoadAcquisitionFromFile(&tc.Config, nil)
+			dss, err := LoadAcquisitionFromFiles(&tc.Config, nil)
 			cstest.RequireErrorContains(t, err, tc.ExpectedError)
 
 			if tc.ExpectedError != "" {
@@ -321,7 +322,8 @@ func TestLoadAcquisitionFromFile(t *testing.T) {
 
 type MockCat struct {
 	configuration.DataSourceCommonCfg `yaml:",inline"`
-	logger                            *log.Entry
+
+	logger *log.Entry
 }
 
 func (f *MockCat) Configure(cfg []byte, logger *log.Entry, metricsLevel int) error {
@@ -366,7 +368,8 @@ func (f *MockCat) GetUuid() string { return "" }
 
 type MockTail struct {
 	configuration.DataSourceCommonCfg `yaml:",inline"`
-	logger                            *log.Entry
+
+	logger *log.Entry
 }
 
 func (f *MockTail) Configure(cfg []byte, logger *log.Entry, metricsLevel int) error {
@@ -421,11 +424,12 @@ func TestStartAcquisitionCat(t *testing.T) {
 
 	go func() {
 		if err := StartAcquisition(ctx, sources, out, &acquisTomb); err != nil {
-			t.Errorf("unexpected error")
+			t.Error("unexpected error")
 		}
 	}()
 
 	count := 0
+
 READLOOP:
 	for {
 		select {
@@ -449,11 +453,12 @@ func TestStartAcquisitionTail(t *testing.T) {
 
 	go func() {
 		if err := StartAcquisition(ctx, sources, out, &acquisTomb); err != nil {
-			t.Errorf("unexpected error")
+			t.Error("unexpected error")
 		}
 	}()
 
 	count := 0
+
 READLOOP:
 	for {
 		select {
@@ -502,6 +507,7 @@ func TestStartAcquisitionTailError(t *testing.T) {
 	}()
 
 	count := 0
+
 READLOOP:
 	for {
 		select {
@@ -511,6 +517,7 @@ READLOOP:
 			break READLOOP
 		}
 	}
+
 	assert.Equal(t, 10, count)
 	// acquisTomb.Kill(nil)
 	time.Sleep(1 * time.Second)
@@ -519,8 +526,9 @@ READLOOP:
 
 type MockSourceByDSN struct {
 	configuration.DataSourceCommonCfg `yaml:",inline"`
-	Toto                              string     `yaml:"toto"`
-	logger                            *log.Entry //nolint: unused
+
+	Toto   string     `yaml:"toto"`
+	logger *log.Entry //nolint: unused
 }
 
 func (f *MockSourceByDSN) UnmarshalConfig(cfg []byte) error { return nil }