diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000000..b226834ed98 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,5 @@ +# Under windows, don't mess with line endings at least for yaml test files, +# otherwise the reported line numbers don't match the expected errors. + +**/testdata/**/*.yaml text eol=lf +**/testdata/**/*.yml text eol=lf diff --git a/cmd/crowdsec-cli/clisetup/setup/detect_test.go b/cmd/crowdsec-cli/clisetup/setup/detect_test.go index aeb7a453ff4..832566d3932 100644 --- a/cmd/crowdsec-cli/clisetup/setup/detect_test.go +++ b/cmd/crowdsec-cli/clisetup/setup/detect_test.go @@ -345,29 +345,6 @@ detect: datasource: {}`, want: nil, wantErr: "invalid acquisition spec for wizard: datasource configuration is empty", - }, { - name: "missing acquisition file name", - config: ` -detect: - wizard: - acquisition_spec: - filename: something.yaml - datasource: - labels: - type: something`, - want: nil, - wantErr: "invalid acquisition spec for wizard: source field is required", - }, { - name: "source is unknown", - config: ` -detect: - foobar: - acquisition_spec: - filename: wombat.yaml - datasource: - source: wombat`, - want: nil, - wantErr: "invalid acquisition spec for foobar: unknown data source wombat", }, { name: "source is misplaced", config: ` @@ -379,131 +356,6 @@ detect: source: file`, want: nil, wantErr: "yaml: unmarshal errors:\n line 7: field source not found in type setup.AcquisitionSpec", - }, { - name: "source is mismatched", - config: ` -detect: - foobar: - acquisition_spec: - filename: journalctl.yaml - datasource: - source: journalctl - filename: /path/to/file.log`, - want: nil, - wantErr: `invalid acquisition spec for foobar: cannot parse: [1:1] unknown field "filename"`, - }, { - name: "source file: required fields", - config: ` -detect: - foobar: - acquisition_spec: - filename: file.yaml - datasource: - source: file`, - want: nil, - wantErr: "invalid acquisition spec for foobar: no filename or filenames configuration provided", - }, { - name: "source journalctl: required fields", - config: ` -detect: - foobar: - acquisition_spec: - filename: foobar.yaml - datasource: - source: journalctl`, - want: nil, - wantErr: "invalid acquisition spec for foobar: journalctl_filter is required", - }, { - name: "source cloudwatch: required fields", - config: ` -detect: - foobar: - acquisition_spec: - filename: cloudwatch.yaml - datasource: - source: cloudwatch`, - want: nil, - wantErr: "invalid acquisition spec for foobar: group_name is mandatory for CloudwatchSource", - }, { - name: "source syslog: all fields are optional", - config: ` -detect: - foobar: - acquisition_spec: - filename: syslog.yaml - datasource: - source: syslog`, - want: &Setup{ - Plans: []ServicePlan{ - { - Name: "foobar", - InstallRecommendation: InstallRecommendation{ - AcquisitionSpec: AcquisitionSpec{ - Filename: "syslog.yaml", - Datasource: DatasourceConfig{ - "source": "syslog", - }, - }, - }, - }, - }, - }, - }, { - name: "source docker: required fields", - config: ` -detect: - foobar: - acquisition_spec: - filename: docker.yaml - datasource: - source: docker`, - want: nil, - wantErr: "invalid acquisition spec for foobar: no containers or services configuration provided", - }, { - name: "source kinesis: required fields (enhanced fanout=false)", - config: ` -detect: - foobar: - acquisition_spec: - filename: kinesis.yaml - datasource: - source: kinesis`, - want: nil, - wantErr: "invalid acquisition spec for foobar: stream_name is mandatory when use_enhanced_fanout is false", - }, { - name: "source kinesis: required fields (enhanced fanout=true)", - config: ` -detect: - foobar: - acquisition_spec: - filename: kinesis.yaml - datasource: - source: kinesis - use_enhanced_fanout: true`, - want: nil, - wantErr: "invalid acquisition spec for foobar: stream_arn is mandatory when use_enhanced_fanout is true", - }, { - name: "source kafka: required fields", - config: ` -detect: - foobar: - acquisition_spec: - filename: kafka.yaml - datasource: - source: kafka`, - want: nil, - wantErr: "invalid acquisition spec for foobar: cannot create a kafka reader with an empty list of broker addresses", - }, { - name: "source loki: required fields", - config: ` -detect: - foobar: - acquisition_spec: - filename: loki.yaml - datasource: - source: loki`, - want: nil, - wantErr: "invalid acquisition spec for foobar: loki query is mandatory", }, } diff --git a/go.mod b/go.mod index 9632299d3eb..c8524364a5a 100644 --- a/go.mod +++ b/go.mod @@ -71,6 +71,7 @@ require ( github.com/prometheus/common v0.66.1 github.com/r3labs/diff/v2 v2.15.1 github.com/sanity-io/litter v1.5.8 + github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 github.com/segmentio/kafka-go v0.4.48 github.com/shirou/gopsutil/v4 v4.25.8 github.com/sirupsen/logrus v1.9.3 diff --git a/go.sum b/go.sum index 50ac03023c0..57ac167a41c 100644 --- a/go.sum +++ b/go.sum @@ -148,6 +148,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= +github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94= github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -508,6 +510,8 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sanity-io/litter v1.5.8 h1:uM/2lKrWdGbRXDrIq08Lh9XtVYoeGtcQxk9rtQ7+rYg= github.com/sanity-io/litter v1.5.8/go.mod h1:9gzJgR2i4ZpjZHsKvUXIRQVk7P+yM3e+jAF7bU2UI5U= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/segmentio/kafka-go v0.4.48 h1:9jyu9CWK4W5W+SroCe8EffbrRZVqAOkuaLd/ApID4Vs= github.com/segmentio/kafka-go v0.4.48/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg= diff --git a/pkg/acquisition/acquisition.go b/pkg/acquisition/acquisition.go index 9131522b757..a162f48d1e3 100644 --- a/pkg/acquisition/acquisition.go +++ b/pkg/acquisition/acquisition.go @@ -10,6 +10,7 @@ import ( "os" "time" "slices" + "strconv" "strings" "github.com/cenkalti/backoff/v5" @@ -89,6 +90,9 @@ func DataSourceConfigure( if lapiClientAware, ok := dataSrc.(types.LAPIClientAware); ok { cConfig := csconfig.GetConfig() + if cConfig.API == nil { + return nil, errors.New("crowdsec configuration not loaded while initializing appsec - this is a bug, plese report") + } lapiClientAware.SetClientConfig(cConfig.API.Client) } @@ -146,7 +150,7 @@ func LoadAcquisitionFromDSN( subLogger := log.StandardLogger().WithField("type", labels["type"]) if err = dsnConf.ConfigureByDSN(ctx, dsn, labels, subLogger, uniqueID); err != nil { - return nil, fmt.Errorf("configuring datasource for %q: %w", dsn, err) + return nil, fmt.Errorf("datasource for %q: %w", dsn, err) } return dataSrc, nil @@ -202,6 +206,115 @@ func detectType(r io.Reader) (string, error) { } } +type ParsedSourceConfig struct { + Common configuration.DataSourceCommonCfg + Source types.DataSource + Transform *vm.Program + SourceMissing bool // the "source" field was missing, and detected + SourceOverridden string // the "source" field was not missing, but didn't match the detected one +} + +var ErrEmptyYAMLDocument = errors.New("empty yaml document") + +// ParseSourceConfig validates and configures one YAML document. +// +// It does not expand env variables, they must already be expanded. +// +// - return sentinel error for empty/comment-only documents +// - backward-compat source auto-detection (filename/filenames/journalctl_filter) +// - validate common fields +// - delegate per-source config validation to the appropriate module +// - compile transform expression +func ParseSourceConfig(ctx context.Context, yamlDoc []byte, metricsLevel metrics.AcquisitionMetricsLevel, hub *cwhub.Hub) (*ParsedSourceConfig, error) { + detectedType, err := detectType(bytes.NewReader(yamlDoc)) + if err != nil { + return nil, err + } + + // if there are not keys or only comments, the document will be skipped + empty, err := csyaml.IsEmptyYAML(bytes.NewReader(yamlDoc)) + if err != nil { + return nil, err + } + + if empty { + return nil, ErrEmptyYAMLDocument + } + + var sub configuration.DataSourceCommonCfg + + // can't be strict here, the doc contains specific datasource config too but we won't collect them now. + if err = yaml.UnmarshalWithOptions(yamlDoc, &sub); err != nil { + return nil, fmt.Errorf("failed to parse: %w", errors.New(yaml.FormatError(err, false, false))) + } + + parsed := &ParsedSourceConfig{} + + // report that the user did not specify a source + if sub.Source == "" { + parsed.SourceMissing = true + } + + // report that the user specified a source that doesn't match with one detected from the presence of other fields + if detectedType != "" { + if sub.Source != "" && sub.Source != detectedType { + parsed.SourceOverridden = sub.Source + } + + sub.Source = detectedType + } + + parsed.Common = sub + + // could not detect, alas + if sub.Source == "" { + return nil, errors.New("missing 'source' field") + } + + // pre-check that the source is valid + _, err = registry.LookupFactory(sub.Source) + if err != nil { + return nil, err + } + + // check for labels now, an error for missing labels has lower priority + // than missing or unknown source type + if len(sub.Labels) == 0 && sub.Source != "docker" { + // docker is the only source that does not require labels + return nil, errors.New("missing labels") + } + + uniqueID := uuid.NewString() + sub.UniqueId = uniqueID + + src, err := DataSourceConfigure(ctx, sub, yamlDoc, metricsLevel, hub) + if err != nil { + return nil, fmt.Errorf("datasource of type %s: %w", sub.Source, err) + } + parsed.Source = src + + if sub.TransformExpr != "" { + vm, err := expr.Compile(sub.TransformExpr, exprhelpers.GetExprOptions(map[string]any{"evt": &pipeline.Event{}})...) + if err != nil { + return nil, fmt.Errorf("while compiling transform expression '%s' for datasource %s: %w", sub.TransformExpr, sub.Source, err) + } + + parsed.Transform = vm + } + + return parsed, nil +} + +func formatConfigLocation(acquisFile string, withPos bool, idx int) string { + ret := acquisFile + + if withPos { + ret += " (position " + strconv.Itoa(idx) + ")" + } + + return ret +} + // sourcesFromFile reads and parses one acquisition file into DataSources. func sourcesFromFile( ctx context.Context, @@ -235,84 +348,42 @@ func sourcesFromFile( idx := -1 for _, yamlDoc := range documents { - detectedType, err := detectType(bytes.NewReader(yamlDoc)) - if err != nil { - return nil, fmt.Errorf("failed to parse %s: %w", yamlFile.Name(), err) - } - idx += 1 - var sub configuration.DataSourceCommonCfg - - // can't be strict here, the doc contains specific datasource config too but we won't collect them now. - if err = yaml.UnmarshalWithOptions(yamlDoc, &sub); err != nil { - return nil, fmt.Errorf("failed to parse %s: %w", yamlFile.Name(), errors.New(yaml.FormatError(err, false, false))) - } + loc := formatConfigLocation(acquisFile, len(documents) > 1, idx) - // for backward compat ('type' was not mandatory, detect it) - if guessType := detectedType; guessType != "" { - log.Debugf("datasource type missing in %s (position %d): detected 'source=%s'", acquisFile, idx, guessType) + parsed, err := ParseSourceConfig(ctx, yamlDoc, metricsLevel, hub) - if sub.Source != "" && sub.Source != guessType { - log.Warnf("datasource type mismatch in %s (position %d): found '%s' but should probably be '%s'", acquisFile, idx, sub.Source, guessType) + // report data source detection, it can be required to understand an error + if parsed != nil { + if parsed.SourceMissing { + log.Debugf("%s: datasource type missing, detected 'source=%s'", loc, parsed.Common.Source) } - sub.Source = guessType - } - - // it's an empty item, skip it - - empty, err := csyaml.IsEmptyYAML(bytes.NewReader(yamlDoc)) - if err != nil { - return nil, fmt.Errorf("failed to parse %s (position %d): %w", acquisFile, idx, err) - } - - if empty { - // there are no keys or only comments, skip the document - continue - } - - if len(sub.Labels) == 0 { - if sub.Source != "docker" { - // docker is the only source that can be empty - return nil, fmt.Errorf("missing labels in %s (position %d)", acquisFile, idx) + if parsed.SourceOverridden != "" { + log.Warnf("%s: datasource type mismatch: found '%s' but should probably be '%s'", loc, parsed.SourceOverridden, parsed.Common.Source) } } - if sub.Source == "" { - return nil, fmt.Errorf("missing 'source' field in %s (position %d)", acquisFile, idx) - } - - // pre-check that the source is valid - _, err = registry.LookupFactory(sub.Source) if err != nil { - return nil, fmt.Errorf("in file %s (position %d) - %w", acquisFile, idx, err) - } - - uniqueID := uuid.NewString() - sub.UniqueId = uniqueID + if errors.Is(err, ErrEmptyYAMLDocument) { + continue + } - src, err := DataSourceConfigure(ctx, sub, yamlDoc, metricsLevel, hub) - if err != nil { var dserr *DataSourceUnavailableError if errors.As(err, &dserr) { - log.Error(err) + log.Error(fmt.Errorf("%s: %w", loc, err)) continue } - return nil, fmt.Errorf("configuring datasource of type %s from %s (position %d): %w", sub.Source, acquisFile, idx, err) + return nil, fmt.Errorf("%s: %w", loc, err) } - if sub.TransformExpr != "" { - vm, err := expr.Compile(sub.TransformExpr, exprhelpers.GetExprOptions(map[string]any{"evt": &pipeline.Event{}})...) - if err != nil { - return nil, fmt.Errorf("while compiling transform expression '%s' for datasource %s in %s (position %d): %w", sub.TransformExpr, sub.Source, acquisFile, idx, err) - } - - transformRuntimes[uniqueID] = vm + if parsed.Transform != nil { + transformRuntimes[parsed.Common.UniqueId] = parsed.Transform } - sources = append(sources, src) + sources = append(sources, parsed.Source) } return sources, nil diff --git a/pkg/acquisition/acquisition_test.go b/pkg/acquisition/acquisition_test.go index 56685f345ef..51dc98780b1 100644 --- a/pkg/acquisition/acquisition_test.go +++ b/pkg/acquisition/acquisition_test.go @@ -264,7 +264,7 @@ func TestLoadAcquisitionFromFiles(t *testing.T) { Config: csconfig.CrowdsecServiceCfg{ AcquisitionFiles: []string{"testdata/missing_labels.yaml"}, }, - ExpectedError: "missing labels in testdata/missing_labels.yaml", + ExpectedError: "testdata/missing_labels.yaml: missing labels", }, { TestName: "backward_compat", @@ -278,14 +278,14 @@ func TestLoadAcquisitionFromFiles(t *testing.T) { Config: csconfig.CrowdsecServiceCfg{ AcquisitionFiles: []string{"testdata/bad_source.yaml"}, }, - ExpectedError: "in file testdata/bad_source.yaml (position 0) - unknown data source does_not_exist", + ExpectedError: "testdata/bad_source.yaml: unknown data source does_not_exist", }, { TestName: "invalid_filetype_config", Config: csconfig.CrowdsecServiceCfg{ AcquisitionFiles: []string{"testdata/bad_filetype.yaml"}, }, - ExpectedError: "configuring datasource of type file from testdata/bad_filetype.yaml", + ExpectedError: "testdata/bad_filetype.yaml: datasource of type file: cannot parse FileAcquisition configuration: [2:12] string was used where sequence is expected", }, { TestName: "from_env", diff --git a/pkg/acquisition/config_test.go b/pkg/acquisition/config_test.go new file mode 100644 index 00000000000..af8312e293e --- /dev/null +++ b/pkg/acquisition/config_test.go @@ -0,0 +1,185 @@ +package acquisition + +import ( + "errors" + "io/fs" + "os" + "path/filepath" + "regexp" + "runtime" + "slices" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/goccy/go-yaml" + + "github.com/crowdsecurity/crowdsec/pkg/csconfig" + "github.com/crowdsecurity/crowdsec/pkg/cwhub" + "github.com/crowdsecurity/crowdsec/pkg/metrics" +) + +var ( + wantErrLineRE = regexp.MustCompile(`(?m)^\s*#\s*wantErr:\s*(.*?)\s*$`) + wantSchemaErrLineRE = regexp.MustCompile(`(?m)^[ \t]*#[ \t]*schemaErr:[ \t]*([^\r\n]*)[ \t]*$`) +) + +func findYAMLFiles(t *testing.T, root string) []string { + t.Helper() + + var files []string + err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + ext := strings.ToLower(filepath.Ext(d.Name())) + if ext != ".yaml" && ext != ".yml" { + return nil + } + files = append(files, path) + return nil + }) + require.NoError(t, err, "walking %q", root) + + slices.Sort(files) + return files +} + +func wantErrFromYAML(t *testing.T, fileContent []byte) (want string, found bool) { + t.Helper() + + m := wantErrLineRE.FindSubmatch(fileContent) + if len(m) == 0 { + return "", false + } + + return strings.TrimSpace(string(m[1])), true +} + +func wantSchemaErrFromYAML(t *testing.T, fileContent []byte) (want string, found bool) { + t.Helper() + + m := wantSchemaErrLineRE.FindSubmatch(fileContent) + if len(m) == 0 { + return "", false + } + + return strings.TrimSpace(string(m[1])), true +} + +func TestParseSourceConfig(t *testing.T) { + ctx := t.Context() + + type suite struct { + name string + root string + expectValid bool + } + + type source struct { + Source string + } + + // load a configuration, appsec needs it + _, _, err := csconfig.NewConfig("./testdata/config.yaml", false, false, true) + require.NoError(t, err) + + // load a hub, appsec needs it + hub := cwhub.Hub{} + + suites := []suite{ + {name: "valid", root: filepath.Join("testdata", "valid"), expectValid: true}, + {name: "invalid", root: filepath.Join("testdata", "invalid"), expectValid: false}, + } + + for _, s := range suites { + t.Run(s.name, func(t *testing.T) { + files := findYAMLFiles(t, s.root) + require.NotEmpty(t, files, "no YAML files found under %q", s.root) + + for _, path := range files { + rel, _ := filepath.Rel(s.root, path) + + fileContent, err := os.ReadFile(path) + require.NoError(t, err, "read %q", path) + + t.Run(filepath.ToSlash(rel), func(t *testing.T) { + var ( + so source + schema string + ) + + if err = yaml.Unmarshal(fileContent, &so); err == nil { + schema = filepath.Join("schemas", so.Source + ".yaml") + } + + if runtime.GOOS == "windows" && strings.Contains(path, "journalctl") { + return + } + + if runtime.GOOS != "windows" && strings.Contains(path, "wineventlog") { + return + } + + wantErr, hasWant := wantErrFromYAML(t, fileContent) + + wantSchemaErr, hasWantSchemaErr := wantSchemaErrFromYAML(t, fileContent) + + if s.expectValid { + require.False(t, hasWant, "valid config must not include # wantErr: directive") + parsed, err := ParseSourceConfig(ctx, fileContent, metrics.AcquisitionMetricsLevelNone, &hub) + require.NoError(t, err) + require.NotNil(t, parsed) + if schema != "" { + err = ValidateYAML(fileContent, schema) + if !errors.Is(err, fs.ErrNotExist) { + // XXX: ignore missing schema + require.NoError(t, err) + } + } + return + } + + // invalid + + require.True(t, hasWant, "invalid config must include '# wantErr: '") + require.NotEmpty(t, wantErr, "wantErr directive found but empty") + + parsed, err := ParseSourceConfig(ctx, fileContent, metrics.AcquisitionMetricsLevelNone, &hub) + require.Error(t, err, "got no error, expected %q", wantErr) + require.Nil(t, parsed) + assert.Equal(t, wantErr, err.Error()) + if schema == "" { + return + } + + // schema validation + + err = ValidateYAML(fileContent, schema) + if errors.Is(err, fs.ErrNotExist) { + // XXX: ignore missing schema, for now + return + } + + // a "schemaErr" comment must be present, even if empty + require.True(t, hasWantSchemaErr, "invalid configurations require an exlicit schemaErr comment. it can be empty string if the schema cannot detect the issue") + switch { + case err == nil && wantSchemaErr != "": + require.Error(t, err, "got no schema error, expected %q", wantSchemaErr) + case err != nil && wantSchemaErr == "": + require.Error(t, err, "got schema error %q, expected nil", err) + case err != nil: + assert.Contains(t, err.Error(), wantSchemaErr) + default: + require.NoError(t, err) + assert.Empty(t, wantSchemaErr) + } + }) + } + }) + } +} diff --git a/pkg/acquisition/modules/appsec/config.go b/pkg/acquisition/modules/appsec/config.go index 149d6217b51..4b9554faa64 100644 --- a/pkg/acquisition/modules/appsec/config.go +++ b/pkg/acquisition/modules/appsec/config.go @@ -128,6 +128,10 @@ func (w *Source) Configure(_ context.Context, yamlConfig []byte, logger *log.Ent return fmt.Errorf("unable to parse appsec configuration: %w", err) } + if w.lapiClientConfig.Credentials == nil { + return errors.New("missing lapi client credentials") + } + w.lapiURL = fmt.Sprintf("%sv1/decisions/stream", w.lapiClientConfig.Credentials.URL) w.AuthCache = NewAuthCache() diff --git a/pkg/acquisition/modules/cloudwatch/cloudwatch_test.go b/pkg/acquisition/modules/cloudwatch/cloudwatch_test.go index 34e7e646a5e..4532bc5d759 100644 --- a/pkg/acquisition/modules/cloudwatch/cloudwatch_test.go +++ b/pkg/acquisition/modules/cloudwatch/cloudwatch_test.go @@ -422,25 +422,6 @@ group_name: test_group stream_name: test_stream`, expectedStartErr: "The specified log group does not exist", }, - { - config: ` -xxx: cloudwatch -labels: - type: test_source -group_name: test_group -stream_name: test_stream`, - expectedCfgErr: `[2:1] unknown field "xxx"`, - }, - { - name: "missing_group_name", - config: ` -source: cloudwatch -aws_region: us-east-1 -labels: - type: test_source -stream_name: test_stream`, - expectedCfgErr: "group_name is mandatory for CloudwatchSource", - }, } for _, tc := range tests { diff --git a/pkg/acquisition/modules/docker/docker_test.go b/pkg/acquisition/modules/docker/docker_test.go index f9a8666bf3d..a353b0c1684 100644 --- a/pkg/acquisition/modules/docker/docker_test.go +++ b/pkg/acquisition/modules/docker/docker_test.go @@ -30,113 +30,6 @@ import ( const testContainerName = "docker_test" const testServiceName = "test_service" -func TestConfigure(t *testing.T) { - log.Infof("Test 'TestConfigure'") - - ctx := t.Context() - - tests := []struct { - config string - expectedErr string - }{ - { - config: `foobar: asd`, - expectedErr: `while parsing DockerAcquisition configuration: [1:1] unknown field "foobar"`, - }, - { - config: ` -mode: tail -source: docker`, - expectedErr: "no containers or services configuration provided", - }, - { - config: ` -mode: cat -source: docker -container_name: - - toto`, - expectedErr: "", - }, - { - config: ` -mode: cat -source: docker -check_interval: 10s -container_name: - - toto`, - expectedErr: "", - }, - { - config: ` -mode: cat -source: docker -service_name: - - web-service`, - expectedErr: "", - }, - { - config: ` -mode: cat -source: docker -use_container_labels: true -container_name: - - toto`, - expectedErr: "use_container_labels and container_name, container_id, container_id_regexp, container_name_regexp are mutually exclusive", - }, - { - config: ` -mode: cat -source: docker -use_service_labels: true -service_name: - - web-service`, - expectedErr: "use_service_labels and service_name, service_id, service_id_regexp, service_name_regexp are mutually exclusive", - }, - { - config: ` -mode: cat -source: docker -container_name_regexp: - - "[invalid"`, - expectedErr: "container_name_regexp: error parsing regexp: missing closing ]: `[invalid`", - }, - { - config: ` -mode: cat -source: docker -container_id_regexp: - - "*invalid"`, - expectedErr: "container_id_regexp: error parsing regexp: missing argument to repetition operator: `*`", - }, - { - config: ` -mode: cat -source: docker -service_name_regexp: - - "(?P - {server="demo"} -`, - expectedErr: "", - testName: "Correct config", - }, - { - config: ` -mode: tail -source: loki -url: http://localhost:3100/ -wait_for_ready: 5s -query: > - {server="demo"} -`, - expectedErr: "", - testName: "Correct config with wait_for_ready", - waitForReady: 5 * time.Second, - }, - { - config: ` -mode: tail -source: loki -url: http://localhost:3100/ -delay_for: 1s -query: > - {server="demo"} -`, - expectedErr: "", - testName: "Correct config with delay_for", - delayFor: 1 * time.Second, - }, - { - config: ` -mode: tail -source: loki -url: http://localhost:3100/ -no_ready_check: true -query: > - {server="demo"} -`, - expectedErr: "", - testName: "Correct config with no_ready_check", - noReadyCheck: true, - }, - { - config: ` -mode: tail -source: loki -url: http://localhost:3100/ -auth: - username: foo - password: bar -query: > - {server="demo"} -`, - expectedErr: "", - password: "bar", - testName: "Correct config with password", - }, - { - config: ` -mode: tail -source: loki -url: http://localhost:3100/ -delay_for: 10s -query: > - {server="demo"} -`, - expectedErr: "delay_for should be a value between 1s and 5s", - testName: "Invalid DelayFor", - }, - { - config: ` -source: loki -no_ready_check: 37 -`, - expectedErr: "[3:17] cannot unmarshal uint64 into Go struct field Configuration.NoReadyCheck of type bool", - testName: "type mismatch", - }, - } - subLogger := log.WithField("type", loki.ModuleName) - - for _, test := range tests { - t.Run(test.testName, func(t *testing.T) { - lokiSource := loki.Source{} - err := lokiSource.Configure(ctx, []byte(test.config), subLogger, metrics.AcquisitionMetricsLevelNone) - cstest.AssertErrorContains(t, err, test.expectedErr) - - if test.password != "" { - p := lokiSource.Config.Auth.Password - if test.password != p { - t.Fatalf("Password mismatch : %s != %s", test.password, p) - } - } - - if test.waitForReady != 0 { - if lokiSource.Config.WaitForReady != test.waitForReady { - t.Fatalf("Wrong WaitForReady %v != %v", lokiSource.Config.WaitForReady, test.waitForReady) - } - } - - if test.delayFor != 0 { - if lokiSource.Config.DelayFor != test.delayFor { - t.Fatalf("Wrong DelayFor %v != %v", lokiSource.Config.DelayFor, test.delayFor) - } - } - - assert.Equal(t, test.noReadyCheck, lokiSource.Config.NoReadyCheck) - }) - } -} - func TestConfigureDSN(t *testing.T) { log.Infof("Test 'TestConfigureDSN'") diff --git a/pkg/acquisition/modules/s3/s3_test.go b/pkg/acquisition/modules/s3/s3_test.go index 6be11659f9e..8557a8fa37e 100644 --- a/pkg/acquisition/modules/s3/s3_test.go +++ b/pkg/acquisition/modules/s3/s3_test.go @@ -19,118 +19,10 @@ import ( "github.com/stretchr/testify/require" "gopkg.in/tomb.v2" - "github.com/crowdsecurity/go-cs-lib/cstest" - "github.com/crowdsecurity/crowdsec/pkg/metrics" "github.com/crowdsecurity/crowdsec/pkg/pipeline" ) -func TestBadConfiguration(t *testing.T) { - ctx := t.Context() - - tests := []struct { - name string - config string - expectedErr string - }{ - { - name: "no bucket", - config: ` -source: s3 -`, - expectedErr: "bucket_name is required", - }, - { - name: "type mismatch", - config: ` -source: s3 -max_buffer_size: true -`, - expectedErr: "[3:18] cannot unmarshal bool into Go struct field Configuration.MaxBufferSize of type int", - }, - { - name: "invalid polling method", - config: ` -source: s3 -bucket_name: foobar -polling_method: foobar -`, - expectedErr: "invalid polling method foobar", - }, - { - name: "no sqs name", - config: ` -source: s3 -bucket_name: foobar -polling_method: sqs -`, - expectedErr: "sqs_name is required when using sqs polling method", - }, - { - name: "both bucket and sqs", - config: ` -source: s3 -bucket_name: foobar -polling_method: sqs -sqs_name: foobar -`, - expectedErr: "bucket_name and sqs_name are mutually exclusive", - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - f := Source{} - - err := f.Configure(ctx, []byte(test.config), nil, metrics.AcquisitionMetricsLevelNone) - cstest.RequireErrorContains(t, err, test.expectedErr) - }) - } -} - -func TestGoodConfiguration(t *testing.T) { - ctx := t.Context() - - tests := []struct { - name string - config string - }{ - { - name: "basic", - config: ` -source: s3 -bucket_name: foobar -`, - }, - { - name: "polling method", - config: ` -source: s3 -polling_method: sqs -sqs_name: foobar -`, - }, - { - name: "list method", - config: ` -source: s3 -bucket_name: foobar -polling_method: list -`, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - f := Source{} - logger := log.NewEntry(log.New()) - - err := f.Configure(ctx, []byte(test.config), logger, metrics.AcquisitionMetricsLevelNone) - require.NoError(t, err) - }) - } -} - type mockS3Client struct{} // We add one hour to trick the listing goroutine into thinking the files are new diff --git a/pkg/acquisition/modules/syslog/syslog_test.go b/pkg/acquisition/modules/syslog/syslog_test.go index e2c5326eb7b..5d6edab3874 100644 --- a/pkg/acquisition/modules/syslog/syslog_test.go +++ b/pkg/acquisition/modules/syslog/syslog_test.go @@ -18,54 +18,6 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/pipeline" ) -func TestConfigure(t *testing.T) { - ctx := t.Context() - - tests := []struct { - config string - expectedErr string - }{ - { - config: ` -foobar: bla -source: syslog`, - expectedErr: `[2:1] unknown field "foobar"`, - }, - { - config: `source: syslog`, - expectedErr: "", - }, - { - config: ` -source: syslog -listen_port: asd`, - expectedErr: "[3:14] cannot unmarshal string into Go struct field Configuration.Port of type int", - }, - { - config: ` -source: syslog -listen_port: 424242`, - expectedErr: "invalid port 424242", - }, - { - config: ` -source: syslog -listen_addr: 10.0.0`, - expectedErr: "invalid listen IP 10.0.0", - }, - } - - subLogger := log.WithField("type", ModuleName) - - for _, test := range tests { - t.Run(test.config, func(t *testing.T) { - s := Source{} - err := s.Configure(ctx, []byte(test.config), subLogger, metrics.AcquisitionMetricsLevelNone) - cstest.AssertErrorContains(t, err, test.expectedErr) - }) - } -} - func writeToSyslog(ctx context.Context, logs []string) error { dialer := &net.Dialer{} diff --git a/pkg/acquisition/modules/victorialogs/victorialogs_test.go b/pkg/acquisition/modules/victorialogs/victorialogs_test.go index d1cef8d53e8..d082986694a 100644 --- a/pkg/acquisition/modules/victorialogs/victorialogs_test.go +++ b/pkg/acquisition/modules/victorialogs/victorialogs_test.go @@ -26,115 +26,6 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/pipeline" ) -func TestConfiguration(t *testing.T) { - log.Infof("Test 'TestConfigure'") - - ctx := t.Context() - - tests := []struct { - config string - expectedErr string - password string - waitForReady time.Duration - testName string - }{ - { - config: `foobar: asd`, - expectedErr: `[1:1] unknown field "foobar"`, - testName: "Unknown field", - }, - { - config: ` -mode: tail -source: victorialogs`, - expectedErr: "url is mandatory", - testName: "Missing url", - }, - { - config: ` -mode: tail -source: victorialogs -url: http://localhost:9428/ -`, - expectedErr: "query is mandatory", - testName: "Missing query", - }, - { - config: ` -mode: tail -source: victorialogs -url: http://localhost:9428/ -query: > - {server="demo"} -limit: true -`, - expectedErr: "[7:8] cannot unmarshal bool into Go struct field Configuration.Limit of type int", - testName: "mismatched type", - }, - { - config: ` -mode: tail -source: victorialogs -url: http://localhost:9428/ -query: > - {server="demo"} -`, - expectedErr: "", - testName: "Correct config", - }, - { - config: ` -mode: tail -source: victorialogs -url: http://localhost:9428/ -wait_for_ready: 5s -query: > - {server="demo"} -`, - expectedErr: "", - testName: "Correct config with wait_for_ready", - waitForReady: 5 * time.Second, - }, - { - config: ` -mode: tail -source: victorialogs -url: http://localhost:9428/ -auth: - username: foo - password: bar -query: > - {server="demo"} -`, - expectedErr: "", - password: "bar", - testName: "Correct config with password", - }, - } - subLogger := log.WithField("type", victorialogs.ModuleName) - - for _, test := range tests { - t.Run(test.testName, func(t *testing.T) { - vlSource := victorialogs.Source{} - err := vlSource.Configure(ctx, []byte(test.config), subLogger, metrics.AcquisitionMetricsLevelNone) - cstest.AssertErrorContains(t, err, test.expectedErr) - - if test.password != "" { - p := vlSource.Config.Auth.Password - if test.password != p { - t.Fatalf("Password mismatch : %s != %s", test.password, p) - } - } - - if test.waitForReady != 0 { - if vlSource.Config.WaitForReady != test.waitForReady { - t.Fatalf("Wrong WaitForReady %v != %v", vlSource.Config.WaitForReady, test.waitForReady) - } - } - }) - } -} - func TestConfigureDSN(t *testing.T) { log.Infof("Test 'TestConfigureDSN'") diff --git a/pkg/acquisition/modules/wineventlog/wineventlog_windows_test.go b/pkg/acquisition/modules/wineventlog/wineventlog_windows_test.go index 0daf57d2661..7b94228c1a0 100644 --- a/pkg/acquisition/modules/wineventlog/wineventlog_windows_test.go +++ b/pkg/acquisition/modules/wineventlog/wineventlog_windows_test.go @@ -63,7 +63,7 @@ event_ids: true`, }, } - subLogger := log.WithField("type", "windowseventlog") + subLogger := log.WithField("type", ModuleName) for _, test := range tests { f := Source{} err := f.Configure(ctx, []byte(test.config), subLogger, metrics.AcquisitionMetricsLevelNone) @@ -122,7 +122,7 @@ event_level: bla`, expectedErr: "invalid log level", }, } - subLogger := log.WithField("type", "windowseventlog") + subLogger := log.WithField("type", ModuleName) for _, test := range tests { t.Run(test.config, func(t *testing.T) { f := Source{} @@ -190,7 +190,7 @@ event_ids: expectedLines: nil, }, } - subLogger := log.WithField("type", "windowseventlog") + subLogger := log.WithField("type", ModuleName) evthandler, err := eventlog.Open("Application") if err != nil { diff --git a/pkg/acquisition/schemaval_test.go b/pkg/acquisition/schemaval_test.go new file mode 100644 index 00000000000..664698ab962 --- /dev/null +++ b/pkg/acquisition/schemaval_test.go @@ -0,0 +1,102 @@ +package acquisition + +import ( + "bytes" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/santhosh-tekuri/jsonschema/v6" + "github.com/goccy/go-yaml" +) + +// format a compact error without schema location for testing purposes. +func compactSchemaErr(err error) error { + var ve *jsonschema.ValidationError + if !errors.As(err, &ve) { + return err + } + + out := ve.BasicOutput() + if out == nil || len(out.Errors) == 0 { + // Fallback; this may include schema URL, but it's better than losing the error. + return err + } + + msgs := make([]string, 0, len(out.Errors)) + for _, u := range out.Errors { + if u.Error == nil { + continue + } + loc := u.InstanceLocation + if loc == "" { + loc = "/" + } + msgs = append(msgs, fmt.Sprintf("%s: %s", loc, u.Error.String())) + } + + sort.Strings(msgs) + return fmt.Errorf("%s", strings.Join(msgs, "; ")) +} + +// ValidateYAML validates configYAML against schemaPath. +func ValidateYAML(configYAML []byte, schemaPath string) error { + if schemaPath == "" { + return errors.New("no schema provided") + } + + configJSON, err := yaml.YAMLToJSON(configYAML) + if err != nil { + return fmt.Errorf("config: YAML->JSON: %w", err) + } + + configDoc, err := jsonschema.UnmarshalJSON(bytes.NewReader(configJSON)) + if err != nil { + return fmt.Errorf("config: decode JSON: %w", err) + } + + c := jsonschema.NewCompiler() + c.DefaultDraft(jsonschema.Draft2020) + + schemaYAML, err := os.ReadFile(schemaPath) + if err != nil { + return fmt.Errorf("read schema %q: %w", schemaPath, err) + } + + schemaJSON, err := yaml.YAMLToJSON(schemaYAML) + if err != nil { + return fmt.Errorf("schema %q: YAML->JSON: %w", schemaPath, err) + } + + schemaDoc, err := jsonschema.UnmarshalJSON(bytes.NewReader(schemaJSON)) + if err != nil { + return fmt.Errorf("schema %q: decode JSON: %w", schemaPath, err) + } + + abs, err := filepath.Abs(schemaPath) + if err != nil { + return fmt.Errorf("abs %q: %w", schemaPath, err) + } + + if err := c.AddResource(abs, schemaDoc); err != nil { + return fmt.Errorf("add schema resource %q: %w", abs, err) + } + + sch, err := c.Compile(abs) + if err != nil { + return fmt.Errorf("compile schema %q: %w", abs, err) + } + + if err := sch.Validate(configDoc); err != nil { + var ve *jsonschema.ValidationError + if errors.As(err, &ve) { + return compactSchemaErr(ve) + } + return err + } + + return nil +} diff --git a/pkg/acquisition/testdata/config.yaml b/pkg/acquisition/testdata/config.yaml new file mode 100644 index 00000000000..654a3ff6d2c --- /dev/null +++ b/pkg/acquisition/testdata/config.yaml @@ -0,0 +1,6 @@ +# minimal configuration to satisfy appsec +config_paths: + data_dir: /path/to/data +api: + client: + insecure_skip_verify: false diff --git a/pkg/acquisition/testdata/invalid/appsec/appsec_config_required.yaml b/pkg/acquisition/testdata/invalid/appsec/appsec_config_required.yaml new file mode 100644 index 00000000000..08735a7c216 --- /dev/null +++ b/pkg/acquisition/testdata/invalid/appsec/appsec_config_required.yaml @@ -0,0 +1,4 @@ +# wantErr: datasource of type appsec: unable to parse appsec configuration: appsec_config or appsec_config_path must be set +source: appsec +labels: + type: sometype diff --git a/pkg/acquisition/testdata/invalid/appsec/common_log_level_invalid.yaml b/pkg/acquisition/testdata/invalid/appsec/common_log_level_invalid.yaml new file mode 100644 index 00000000000..955676a863b --- /dev/null +++ b/pkg/acquisition/testdata/invalid/appsec/common_log_level_invalid.yaml @@ -0,0 +1,4 @@ +# wantErr: failed to parse: not a valid logrus Level: "toto" +# schemaErr: /log_level: value must be one of 'panic', 'fatal', 'error', 'warn', 'warning', 'info', 'debug', 'trace' +source: appsec +log_level: toto diff --git a/pkg/acquisition/testdata/invalid/appsec/invalid_appsec_config.yaml b/pkg/acquisition/testdata/invalid/appsec/invalid_appsec_config.yaml new file mode 100644 index 00000000000..b9afcc721d9 --- /dev/null +++ b/pkg/acquisition/testdata/invalid/appsec/invalid_appsec_config.yaml @@ -0,0 +1,6 @@ +# wantErr: datasource of type appsec: unable to parse appsec configuration: cannot parse appsec configuration: [6:6] cannot unmarshal map[string]interface {} into Go struct field Configuration.AppsecConfig of type string +source: appsec +labels: + type: sometype +appsec_config: + foo: bar diff --git a/pkg/acquisition/testdata/invalid/appsec/labels_required.yaml b/pkg/acquisition/testdata/invalid/appsec/labels_required.yaml new file mode 100644 index 00000000000..f340e8f3adb --- /dev/null +++ b/pkg/acquisition/testdata/invalid/appsec/labels_required.yaml @@ -0,0 +1,2 @@ +# wantErr: missing labels +source: appsec diff --git a/pkg/acquisition/testdata/invalid/appsec/xxx.yaml b/pkg/acquisition/testdata/invalid/appsec/xxx.yaml new file mode 100644 index 00000000000..50433bd3125 --- /dev/null +++ b/pkg/acquisition/testdata/invalid/appsec/xxx.yaml @@ -0,0 +1,5 @@ +# wantErr: datasource of type appsec: missing lapi client credentials +source: appsec +labels: + type: sometype +appsec_config: something diff --git a/pkg/acquisition/testdata/invalid/cloudwatch/aws_region_required.yaml b/pkg/acquisition/testdata/invalid/cloudwatch/aws_region_required.yaml new file mode 100644 index 00000000000..802dca51e7f --- /dev/null +++ b/pkg/acquisition/testdata/invalid/cloudwatch/aws_region_required.yaml @@ -0,0 +1,6 @@ +# wantErr: datasource of type cloudwatch: aws_region is not specified, specify it or aws_config_dir +source: cloudwatch +labels: + type: sometype +group_name: testgroup + diff --git a/pkg/acquisition/testdata/invalid/cloudwatch/common_log_level_invalid.yaml b/pkg/acquisition/testdata/invalid/cloudwatch/common_log_level_invalid.yaml new file mode 100644 index 00000000000..0bb10366084 --- /dev/null +++ b/pkg/acquisition/testdata/invalid/cloudwatch/common_log_level_invalid.yaml @@ -0,0 +1,4 @@ +# wantErr: failed to parse: not a valid logrus Level: "toto" +# schemaErr: /log_level: value must be one of 'panic', 'fatal', 'error', 'warn', 'warning', 'info', 'debug', 'trace' +source: cloudwatch +log_level: toto diff --git a/pkg/acquisition/testdata/invalid/cloudwatch/group_name_required.yaml b/pkg/acquisition/testdata/invalid/cloudwatch/group_name_required.yaml new file mode 100644 index 00000000000..cff58079b42 --- /dev/null +++ b/pkg/acquisition/testdata/invalid/cloudwatch/group_name_required.yaml @@ -0,0 +1,5 @@ +# wantErr: datasource of type cloudwatch: group_name is mandatory for CloudwatchSource +source: cloudwatch +labels: + type: sometype + diff --git a/pkg/acquisition/testdata/invalid/cloudwatch/labels_required.yaml b/pkg/acquisition/testdata/invalid/cloudwatch/labels_required.yaml new file mode 100644 index 00000000000..02b7f3deb37 --- /dev/null +++ b/pkg/acquisition/testdata/invalid/cloudwatch/labels_required.yaml @@ -0,0 +1,2 @@ +# wantErr: missing labels +source: cloudwatch diff --git a/pkg/acquisition/testdata/invalid/cloudwatch/unknown_field.yaml b/pkg/acquisition/testdata/invalid/cloudwatch/unknown_field.yaml new file mode 100644 index 00000000000..96053037e36 --- /dev/null +++ b/pkg/acquisition/testdata/invalid/cloudwatch/unknown_field.yaml @@ -0,0 +1,5 @@ +# wantErr: datasource of type cloudwatch: cannot parse: [5:1] unknown field "xxx" +source: cloudwatch +labels: + type: sometype +xxx: dummy diff --git a/pkg/acquisition/testdata/invalid/docker/common_log_level_invalid.yaml b/pkg/acquisition/testdata/invalid/docker/common_log_level_invalid.yaml new file mode 100644 index 00000000000..a726148e7a3 --- /dev/null +++ b/pkg/acquisition/testdata/invalid/docker/common_log_level_invalid.yaml @@ -0,0 +1,4 @@ +# wantErr: failed to parse: not a valid logrus Level: "toto" +# schemaErr: /log_level: value must be one of 'panic', 'fatal', 'error', 'warn', 'warning', 'info', 'debug', 'trace' +source: docker +log_level: toto diff --git a/pkg/acquisition/testdata/invalid/docker/common_mode_invalid.yaml b/pkg/acquisition/testdata/invalid/docker/common_mode_invalid.yaml new file mode 100644 index 00000000000..80050a6b5fd --- /dev/null +++ b/pkg/acquisition/testdata/invalid/docker/common_mode_invalid.yaml @@ -0,0 +1,6 @@ +# wantErr: datasource of type docker: unsupported mode server for docker datasource +# schemaErr: /mode: value must be one of 'tail', 'cat' +mode: server +source: docker +container_name: + - toto diff --git a/pkg/acquisition/testdata/invalid/docker/container_id_regexp_parsing_error.yaml b/pkg/acquisition/testdata/invalid/docker/container_id_regexp_parsing_error.yaml new file mode 100644 index 00000000000..735fac1514f --- /dev/null +++ b/pkg/acquisition/testdata/invalid/docker/container_id_regexp_parsing_error.yaml @@ -0,0 +1,5 @@ +# wantErr: datasource of type docker: container_id_regexp: error parsing regexp: missing argument to repetition operator: `*` +# schemaErr: +source: docker +container_id_regexp: + - "*invalid" diff --git a/pkg/acquisition/testdata/invalid/docker/container_labels_mutual_exclusive.yaml b/pkg/acquisition/testdata/invalid/docker/container_labels_mutual_exclusive.yaml new file mode 100644 index 00000000000..875b083a2b7 --- /dev/null +++ b/pkg/acquisition/testdata/invalid/docker/container_labels_mutual_exclusive.yaml @@ -0,0 +1,6 @@ +# wantErr: datasource of type docker: use_container_labels and container_name, container_id, container_id_regexp, container_name_regexp are mutually exclusive +# schemaErr: /: 'allOf' failed; /: 'not' failed +source: docker +use_container_labels: true +container_name: + - toto diff --git a/pkg/acquisition/testdata/invalid/docker/container_name_regexp_parsing_error.yaml b/pkg/acquisition/testdata/invalid/docker/container_name_regexp_parsing_error.yaml new file mode 100644 index 00000000000..2565fc6a00b --- /dev/null +++ b/pkg/acquisition/testdata/invalid/docker/container_name_regexp_parsing_error.yaml @@ -0,0 +1,5 @@ +# wantErr: datasource of type docker: container_name_regexp: error parsing regexp: missing closing ]: `[invalid` +# schemaErr: +source: docker +container_name_regexp: + - "[invalid" diff --git a/pkg/acquisition/testdata/invalid/docker/containers_services_required.yaml b/pkg/acquisition/testdata/invalid/docker/containers_services_required.yaml new file mode 100644 index 00000000000..8ef4f9c0753 --- /dev/null +++ b/pkg/acquisition/testdata/invalid/docker/containers_services_required.yaml @@ -0,0 +1,3 @@ +# wantErr: datasource of type docker: no containers or services configuration provided +# schemaErr: /: 'allOf' failed; /: 'anyOf' failed; /: missing property 'container_id'; /: missing property 'container_id_regexp'; /: missing property 'container_name'; /: missing property 'container_name_regexp'; /: missing property 'service_id'; /: missing property 'service_id_regexp'; /: missing property 'service_name'; /: missing property 'service_name_regexp'; /: missing property 'use_container_labels'; /: missing property 'use_service_labels' +source: docker diff --git a/pkg/acquisition/testdata/invalid/docker/service_id_regexp_parsing_error.yaml b/pkg/acquisition/testdata/invalid/docker/service_id_regexp_parsing_error.yaml new file mode 100644 index 00000000000..b6fc6ac239e --- /dev/null +++ b/pkg/acquisition/testdata/invalid/docker/service_id_regexp_parsing_error.yaml @@ -0,0 +1,5 @@ +# wantErr: datasource of type docker: service_id_regexp: error parsing regexp: missing argument to repetition operator: `+` +# schemaErr: +source: docker +service_id_regexp: + - "+invalid" diff --git a/pkg/acquisition/testdata/invalid/docker/service_labels_mutual_exclusive.yaml b/pkg/acquisition/testdata/invalid/docker/service_labels_mutual_exclusive.yaml new file mode 100644 index 00000000000..9a990c21193 --- /dev/null +++ b/pkg/acquisition/testdata/invalid/docker/service_labels_mutual_exclusive.yaml @@ -0,0 +1,6 @@ +# wantErr: datasource of type docker: use_service_labels and service_name, service_id, service_id_regexp, service_name_regexp are mutually exclusive +# schemaErr: /: 'allOf' failed; /: 'not' failed +source: docker +use_service_labels: true +service_name: + - web-service diff --git a/pkg/acquisition/testdata/invalid/docker/service_name_regexp_parsing_error.yaml b/pkg/acquisition/testdata/invalid/docker/service_name_regexp_parsing_error.yaml new file mode 100644 index 00000000000..b83ac689ac0 --- /dev/null +++ b/pkg/acquisition/testdata/invalid/docker/service_name_regexp_parsing_error.yaml @@ -0,0 +1,5 @@ +# wantErr: datasource of type docker: service_name_regexp: error parsing regexp: invalid named capture: `(?P + {server="demo"} diff --git a/pkg/acquisition/testdata/invalid/loki/labels_required.yaml b/pkg/acquisition/testdata/invalid/loki/labels_required.yaml new file mode 100644 index 00000000000..8520684e4f3 --- /dev/null +++ b/pkg/acquisition/testdata/invalid/loki/labels_required.yaml @@ -0,0 +1,2 @@ +# wantErr: missing labels +source: loki diff --git a/pkg/acquisition/testdata/invalid/loki/no_ready_check_invalid.yaml b/pkg/acquisition/testdata/invalid/loki/no_ready_check_invalid.yaml new file mode 100644 index 00000000000..af9f52ca0a3 --- /dev/null +++ b/pkg/acquisition/testdata/invalid/loki/no_ready_check_invalid.yaml @@ -0,0 +1,5 @@ +# wantErr: datasource of type loki: cannot parse loki acquisition configuration: [5:17] cannot unmarshal uint64 into Go struct field Configuration.NoReadyCheck of type bool +source: loki +labels: + type: sometype +no_ready_check: 37 diff --git a/pkg/acquisition/testdata/invalid/loki/query_required.yaml b/pkg/acquisition/testdata/invalid/loki/query_required.yaml new file mode 100644 index 00000000000..1044dea1fa6 --- /dev/null +++ b/pkg/acquisition/testdata/invalid/loki/query_required.yaml @@ -0,0 +1,4 @@ +# wantErr: datasource of type loki: loki query is mandatory +source: loki +labels: + type: sometype diff --git a/pkg/acquisition/testdata/invalid/loki/unknown_field.yaml b/pkg/acquisition/testdata/invalid/loki/unknown_field.yaml new file mode 100644 index 00000000000..fdf02875fe2 --- /dev/null +++ b/pkg/acquisition/testdata/invalid/loki/unknown_field.yaml @@ -0,0 +1,5 @@ +# wantErr: datasource of type loki: cannot parse loki acquisition configuration: [5:1] unknown field "foobar" +source: loki +labels: + type: sometype +foobar: asd diff --git a/pkg/acquisition/testdata/invalid/mapping_expected.yaml b/pkg/acquisition/testdata/invalid/mapping_expected.yaml new file mode 100644 index 00000000000..0b1eb3580d3 --- /dev/null +++ b/pkg/acquisition/testdata/invalid/mapping_expected.yaml @@ -0,0 +1,2 @@ +# wantErr: failed to parse: [2:1] string was used where mapping is expected +blahblah diff --git a/pkg/acquisition/testdata/invalid/s3/bucket_name_required.yaml b/pkg/acquisition/testdata/invalid/s3/bucket_name_required.yaml new file mode 100644 index 00000000000..a3383e29281 --- /dev/null +++ b/pkg/acquisition/testdata/invalid/s3/bucket_name_required.yaml @@ -0,0 +1,4 @@ +# wantErr: datasource of type s3: bucket_name is required +source: s3 +labels: + type: sometype diff --git a/pkg/acquisition/testdata/invalid/s3/bucket_name_sqs_name_mutual_exclusive.yaml b/pkg/acquisition/testdata/invalid/s3/bucket_name_sqs_name_mutual_exclusive.yaml new file mode 100644 index 00000000000..a0ef12a43b1 --- /dev/null +++ b/pkg/acquisition/testdata/invalid/s3/bucket_name_sqs_name_mutual_exclusive.yaml @@ -0,0 +1,7 @@ +# wantErr: datasource of type s3: bucket_name and sqs_name are mutually exclusive +source: s3 +labels: + type: sometype +bucket_name: foobar +polling_method: sqs +sqs_name: foobar diff --git a/pkg/acquisition/testdata/invalid/s3/common_log_level_invalid.yaml b/pkg/acquisition/testdata/invalid/s3/common_log_level_invalid.yaml new file mode 100644 index 00000000000..cc32c1ed7d2 --- /dev/null +++ b/pkg/acquisition/testdata/invalid/s3/common_log_level_invalid.yaml @@ -0,0 +1,4 @@ +# wantErr: failed to parse: not a valid logrus Level: "toto" +# schemaErr: /log_level: value must be one of 'panic', 'fatal', 'error', 'warn', 'warning', 'info', 'debug', 'trace' +source: s3 +log_level: toto diff --git a/pkg/acquisition/testdata/invalid/s3/labels_required.yaml b/pkg/acquisition/testdata/invalid/s3/labels_required.yaml new file mode 100644 index 00000000000..0c546c59fb4 --- /dev/null +++ b/pkg/acquisition/testdata/invalid/s3/labels_required.yaml @@ -0,0 +1,2 @@ +# wantErr: missing labels +source: s3 diff --git a/pkg/acquisition/testdata/invalid/s3/max_buffer_size_invalid.yaml b/pkg/acquisition/testdata/invalid/s3/max_buffer_size_invalid.yaml new file mode 100644 index 00000000000..9f9e5536fd1 --- /dev/null +++ b/pkg/acquisition/testdata/invalid/s3/max_buffer_size_invalid.yaml @@ -0,0 +1,5 @@ +# wantErr: datasource of type s3: cannot parse S3Acquisition configuration: [5:18] cannot unmarshal bool into Go struct field Configuration.MaxBufferSize of type int +source: s3 +labels: + type: sometype +max_buffer_size: true diff --git a/pkg/acquisition/testdata/invalid/s3/polling_method_invalid.yaml b/pkg/acquisition/testdata/invalid/s3/polling_method_invalid.yaml new file mode 100644 index 00000000000..44fc1788f63 --- /dev/null +++ b/pkg/acquisition/testdata/invalid/s3/polling_method_invalid.yaml @@ -0,0 +1,6 @@ +# wantErr: datasource of type s3: invalid polling method foobar +source: s3 +labels: + type: sometype +bucket_name: foobar +polling_method: foobar diff --git a/pkg/acquisition/testdata/invalid/s3/sqs_name_required.yaml b/pkg/acquisition/testdata/invalid/s3/sqs_name_required.yaml new file mode 100644 index 00000000000..cf1fcb04998 --- /dev/null +++ b/pkg/acquisition/testdata/invalid/s3/sqs_name_required.yaml @@ -0,0 +1,6 @@ +# wantErr: datasource of type s3: sqs_name is required when using sqs polling method +source: s3 +labels: + type: sometype +bucket_name: foobar +polling_method: sqs diff --git a/pkg/acquisition/testdata/invalid/s3/unknown_field.yaml b/pkg/acquisition/testdata/invalid/s3/unknown_field.yaml new file mode 100644 index 00000000000..0ffb829f3a2 --- /dev/null +++ b/pkg/acquisition/testdata/invalid/s3/unknown_field.yaml @@ -0,0 +1,5 @@ +# wantErr: datasource of type s3: cannot parse S3Acquisition configuration: [5:1] unknown field "foobar" +source: s3 +labels: + type: sometype +foobar: asd.log diff --git a/pkg/acquisition/testdata/invalid/source_required.yaml b/pkg/acquisition/testdata/invalid/source_required.yaml new file mode 100644 index 00000000000..dc6d57af3e5 --- /dev/null +++ b/pkg/acquisition/testdata/invalid/source_required.yaml @@ -0,0 +1,2 @@ +# wantErr: missing 'source' field +key: value diff --git a/pkg/acquisition/testdata/invalid/source_unknown.yaml b/pkg/acquisition/testdata/invalid/source_unknown.yaml new file mode 100644 index 00000000000..f0ac8a27f00 --- /dev/null +++ b/pkg/acquisition/testdata/invalid/source_unknown.yaml @@ -0,0 +1,2 @@ +# wantErr: unknown data source wombat +source: wombat diff --git a/pkg/acquisition/testdata/invalid/syslog/common_log_level_invalid.yaml b/pkg/acquisition/testdata/invalid/syslog/common_log_level_invalid.yaml new file mode 100644 index 00000000000..26354748d54 --- /dev/null +++ b/pkg/acquisition/testdata/invalid/syslog/common_log_level_invalid.yaml @@ -0,0 +1,4 @@ +# wantErr: failed to parse: not a valid logrus Level: "toto" +# schemaErr: /log_level: value must be one of 'panic', 'fatal', 'error', 'warn', 'warning', 'info', 'debug', 'trace' +source: syslog +log_level: toto diff --git a/pkg/acquisition/testdata/invalid/syslog/labels_required.yaml b/pkg/acquisition/testdata/invalid/syslog/labels_required.yaml new file mode 100644 index 00000000000..ef25c6eecfb --- /dev/null +++ b/pkg/acquisition/testdata/invalid/syslog/labels_required.yaml @@ -0,0 +1,2 @@ +# wantErr: missing labels +source: syslog diff --git a/pkg/acquisition/testdata/invalid/syslog/listen_addr_invalid.yaml b/pkg/acquisition/testdata/invalid/syslog/listen_addr_invalid.yaml new file mode 100644 index 00000000000..cbf88be9e04 --- /dev/null +++ b/pkg/acquisition/testdata/invalid/syslog/listen_addr_invalid.yaml @@ -0,0 +1,5 @@ +# wantErr: datasource of type syslog: invalid listen IP 10.0.0 +source: syslog +labels: + type: sometype +listen_addr: 10.0.0 diff --git a/pkg/acquisition/testdata/invalid/syslog/listen_port_invalid.yaml b/pkg/acquisition/testdata/invalid/syslog/listen_port_invalid.yaml new file mode 100644 index 00000000000..a3195551297 --- /dev/null +++ b/pkg/acquisition/testdata/invalid/syslog/listen_port_invalid.yaml @@ -0,0 +1,5 @@ +# wantErr: datasource of type syslog: cannot parse: [5:14] cannot unmarshal string into Go struct field Configuration.Port of type int +source: syslog +labels: + type: sometype +listen_port: asd diff --git a/pkg/acquisition/testdata/invalid/syslog/listen_port_invalid_int.yaml b/pkg/acquisition/testdata/invalid/syslog/listen_port_invalid_int.yaml new file mode 100644 index 00000000000..91d0bf62f5b --- /dev/null +++ b/pkg/acquisition/testdata/invalid/syslog/listen_port_invalid_int.yaml @@ -0,0 +1,5 @@ +# wantErr: datasource of type syslog: invalid port 424242 +source: syslog +labels: + type: sometype +listen_port: 424242 diff --git a/pkg/acquisition/testdata/invalid/syslog/unknown_field.yaml b/pkg/acquisition/testdata/invalid/syslog/unknown_field.yaml new file mode 100644 index 00000000000..9182555523e --- /dev/null +++ b/pkg/acquisition/testdata/invalid/syslog/unknown_field.yaml @@ -0,0 +1,5 @@ +# wantErr: datasource of type syslog: cannot parse: [5:1] unknown field "foobar" +source: syslog +labels: + type: sometype +foobar: asd diff --git a/pkg/acquisition/testdata/invalid/victorialogs/common_log_level_invalid.yaml b/pkg/acquisition/testdata/invalid/victorialogs/common_log_level_invalid.yaml new file mode 100644 index 00000000000..f822fec2182 --- /dev/null +++ b/pkg/acquisition/testdata/invalid/victorialogs/common_log_level_invalid.yaml @@ -0,0 +1,4 @@ +# wantErr: failed to parse: not a valid logrus Level: "toto" +# schemaErr: /log_level: value must be one of 'panic', 'fatal', 'error', 'warn', 'warning', 'info', 'debug', 'trace' +source: victorialogs +log_level: toto diff --git a/pkg/acquisition/testdata/invalid/victorialogs/labels_required.yaml b/pkg/acquisition/testdata/invalid/victorialogs/labels_required.yaml new file mode 100644 index 00000000000..6edb22aee39 --- /dev/null +++ b/pkg/acquisition/testdata/invalid/victorialogs/labels_required.yaml @@ -0,0 +1,2 @@ +# wantErr: missing labels +source: victorialogs diff --git a/pkg/acquisition/testdata/invalid/victorialogs/limit_invalid.yaml b/pkg/acquisition/testdata/invalid/victorialogs/limit_invalid.yaml new file mode 100644 index 00000000000..74b6c60a4ac --- /dev/null +++ b/pkg/acquisition/testdata/invalid/victorialogs/limit_invalid.yaml @@ -0,0 +1,8 @@ +# wantErr: datasource of type victorialogs: cannot parse VictoriaLogs acquisition configuration: [8:8] cannot unmarshal bool into Go struct field Configuration.Limit of type int +source: victorialogs +labels: + type: sometype +url: http://localhost:9428/ +query: > + {server="demo"} +limit: true diff --git a/pkg/acquisition/testdata/invalid/victorialogs/query_mandatory.yaml b/pkg/acquisition/testdata/invalid/victorialogs/query_mandatory.yaml new file mode 100644 index 00000000000..7f930734f35 --- /dev/null +++ b/pkg/acquisition/testdata/invalid/victorialogs/query_mandatory.yaml @@ -0,0 +1,5 @@ +# wantErr: datasource of type victorialogs: VictoriaLogs query is mandatory +source: victorialogs +labels: + type: sometype +url: http://localhost:9428/ diff --git a/pkg/acquisition/testdata/invalid/victorialogs/unknown_field.yaml b/pkg/acquisition/testdata/invalid/victorialogs/unknown_field.yaml new file mode 100644 index 00000000000..e22aa9dfc06 --- /dev/null +++ b/pkg/acquisition/testdata/invalid/victorialogs/unknown_field.yaml @@ -0,0 +1,5 @@ +# wantErr: datasource of type victorialogs: cannot parse VictoriaLogs acquisition configuration: [5:1] unknown field "foobar" +source: victorialogs +labels: + type: sometype +foobar: asd.log diff --git a/pkg/acquisition/testdata/invalid/victorialogs/url_mandatory.yaml b/pkg/acquisition/testdata/invalid/victorialogs/url_mandatory.yaml new file mode 100644 index 00000000000..6429fdfbfe3 --- /dev/null +++ b/pkg/acquisition/testdata/invalid/victorialogs/url_mandatory.yaml @@ -0,0 +1,4 @@ +# wantErr: datasource of type victorialogs: VictoriaLogs url is mandatory +source: victorialogs +labels: + type: sometype diff --git a/pkg/acquisition/testdata/invalid/wineventlog/common_log_level_invalid.yaml b/pkg/acquisition/testdata/invalid/wineventlog/common_log_level_invalid.yaml new file mode 100644 index 00000000000..cb74ce192e2 --- /dev/null +++ b/pkg/acquisition/testdata/invalid/wineventlog/common_log_level_invalid.yaml @@ -0,0 +1,4 @@ +# wantErr: failed to parse: not a valid logrus Level: "toto" +# schemaErr: /log_level: value must be one of 'panic', 'fatal', 'error', 'warn', 'warning', 'info', 'debug', 'trace' +source: wineventlog +log_level: toto diff --git a/pkg/acquisition/testdata/invalid/wineventlog/event_channel_xpath_query_mutual_exclusive.yaml b/pkg/acquisition/testdata/invalid/wineventlog/event_channel_xpath_query_mutual_exclusive.yaml new file mode 100644 index 00000000000..ba28071feca --- /dev/null +++ b/pkg/acquisition/testdata/invalid/wineventlog/event_channel_xpath_query_mutual_exclusive.yaml @@ -0,0 +1,6 @@ +# wantErr: datasource of type wineventlog: event_channel and xpath_query are mutually exclusive +source: wineventlog +labels: + type: sometype +event_channel: foo +xpath_query: test diff --git a/pkg/acquisition/testdata/invalid/wineventlog/event_channel_xpath_query_required.yaml b/pkg/acquisition/testdata/invalid/wineventlog/event_channel_xpath_query_required.yaml new file mode 100644 index 00000000000..9de59511dfa --- /dev/null +++ b/pkg/acquisition/testdata/invalid/wineventlog/event_channel_xpath_query_required.yaml @@ -0,0 +1,4 @@ +# wantErr: datasource of type wineventlog: event_channel or xpath_query must be set +source: wineventlog +labels: + type: sometype diff --git a/pkg/acquisition/testdata/invalid/wineventlog/event_ids_invalid.yaml b/pkg/acquisition/testdata/invalid/wineventlog/event_ids_invalid.yaml new file mode 100644 index 00000000000..63e155158e7 --- /dev/null +++ b/pkg/acquisition/testdata/invalid/wineventlog/event_ids_invalid.yaml @@ -0,0 +1,5 @@ +# wantErr: datasource of type wineventlog: cannot parse wineventlog configuration: [5:12] boolean was used where sequence is expected +source: wineventlog +labels: + type: sometype +event_ids: true diff --git a/pkg/acquisition/testdata/invalid/wineventlog/event_level_invalid.yaml b/pkg/acquisition/testdata/invalid/wineventlog/event_level_invalid.yaml new file mode 100644 index 00000000000..5f00e15d50c --- /dev/null +++ b/pkg/acquisition/testdata/invalid/wineventlog/event_level_invalid.yaml @@ -0,0 +1,6 @@ +# wantErr: datasource of type wineventlog: buildXpathQuery failed: invalid log level +source: wineventlog +labels: + type: sometype +event_channel: Security +event_level: blabla diff --git a/pkg/acquisition/testdata/invalid/wineventlog/labels_required.yaml b/pkg/acquisition/testdata/invalid/wineventlog/labels_required.yaml new file mode 100644 index 00000000000..ce6c5f9144b --- /dev/null +++ b/pkg/acquisition/testdata/invalid/wineventlog/labels_required.yaml @@ -0,0 +1,2 @@ +# wantErr: missing labels +source: wineventlog diff --git a/pkg/acquisition/testdata/invalid/wineventlog/unknown_field.yaml b/pkg/acquisition/testdata/invalid/wineventlog/unknown_field.yaml new file mode 100644 index 00000000000..db34c16a874 --- /dev/null +++ b/pkg/acquisition/testdata/invalid/wineventlog/unknown_field.yaml @@ -0,0 +1,5 @@ +# wantErr: datasource of type wineventlog: cannot parse wineventlog configuration: [5:1] unknown field "foobar" +source: wineventlog +labels: + type: sometype +foobar: asd.log diff --git a/pkg/acquisition/testdata/valid/cloudwatch/minimal.yaml b/pkg/acquisition/testdata/valid/cloudwatch/minimal.yaml new file mode 100644 index 00000000000..dde369ef4a6 --- /dev/null +++ b/pkg/acquisition/testdata/valid/cloudwatch/minimal.yaml @@ -0,0 +1,6 @@ +source: cloudwatch +labels: + type: sometype +group_name: testgroup +aws_region: eu-west-1 + diff --git a/pkg/acquisition/testdata/valid/docker/check_interval.yaml b/pkg/acquisition/testdata/valid/docker/check_interval.yaml new file mode 100644 index 00000000000..b828361c2d4 --- /dev/null +++ b/pkg/acquisition/testdata/valid/docker/check_interval.yaml @@ -0,0 +1,4 @@ +source: docker +check_interval: 10s +container_name: + - toto diff --git a/pkg/acquisition/testdata/valid/docker/minimal.yaml b/pkg/acquisition/testdata/valid/docker/minimal.yaml new file mode 100644 index 00000000000..9bfa0336a2b --- /dev/null +++ b/pkg/acquisition/testdata/valid/docker/minimal.yaml @@ -0,0 +1,3 @@ +source: docker +container_name: + - toto diff --git a/pkg/acquisition/testdata/valid/docker/minimal_cat.yaml b/pkg/acquisition/testdata/valid/docker/minimal_cat.yaml new file mode 100644 index 00000000000..745ab479e18 --- /dev/null +++ b/pkg/acquisition/testdata/valid/docker/minimal_cat.yaml @@ -0,0 +1,4 @@ +mode: cat +source: docker +container_name: + - toto diff --git a/pkg/acquisition/testdata/valid/docker/service_name.yaml b/pkg/acquisition/testdata/valid/docker/service_name.yaml new file mode 100644 index 00000000000..a9d77321e49 --- /dev/null +++ b/pkg/acquisition/testdata/valid/docker/service_name.yaml @@ -0,0 +1,3 @@ +source: docker +service_name: + - web-service diff --git a/pkg/acquisition/testdata/valid/file/discovery_poll.yaml b/pkg/acquisition/testdata/valid/file/discovery_poll.yaml new file mode 100644 index 00000000000..bc5b36a13de --- /dev/null +++ b/pkg/acquisition/testdata/valid/file/discovery_poll.yaml @@ -0,0 +1,7 @@ +source: file +labels: + type: sometype +filenames: + - "tests/test.log" +discovery_poll_enable: true +discovery_poll_interval: "30s" diff --git a/pkg/acquisition/testdata/valid/file/polling_disabled.yaml b/pkg/acquisition/testdata/valid/file/polling_disabled.yaml new file mode 100644 index 00000000000..efcea44e499 --- /dev/null +++ b/pkg/acquisition/testdata/valid/file/polling_disabled.yaml @@ -0,0 +1,7 @@ +source: file +labels: + type: sometype +filenames: + - "tests/test.log" +discovery_poll_enable: false + diff --git a/pkg/acquisition/testdata/valid/journalctl/minimal.yaml b/pkg/acquisition/testdata/valid/journalctl/minimal.yaml new file mode 100644 index 00000000000..564a475e24b --- /dev/null +++ b/pkg/acquisition/testdata/valid/journalctl/minimal.yaml @@ -0,0 +1,5 @@ +source: journalctl +labels: + type: sometype +journalctl_filter: + - _UID=42 diff --git a/pkg/acquisition/testdata/valid/kafka/minimal.yaml b/pkg/acquisition/testdata/valid/kafka/minimal.yaml new file mode 100644 index 00000000000..9e803bea1b5 --- /dev/null +++ b/pkg/acquisition/testdata/valid/kafka/minimal.yaml @@ -0,0 +1,6 @@ +source: kafka +labels: + type: sometype +brokers: + - localhost:9092 +topic: crowdsec diff --git a/pkg/acquisition/testdata/valid/loki/full.yaml b/pkg/acquisition/testdata/valid/loki/full.yaml new file mode 100644 index 00000000000..94bae6bc21d --- /dev/null +++ b/pkg/acquisition/testdata/valid/loki/full.yaml @@ -0,0 +1,12 @@ +source: loki +labels: + type: sometype +url: http://localhost:3100/ +wait_for_ready: 5s +delay_for: 1s +no_ready_check: true +auth: + username: foo + password: bar +query: > + {server="demo"} diff --git a/pkg/acquisition/testdata/valid/loki/minimal.yaml b/pkg/acquisition/testdata/valid/loki/minimal.yaml new file mode 100644 index 00000000000..b40ce0eccb6 --- /dev/null +++ b/pkg/acquisition/testdata/valid/loki/minimal.yaml @@ -0,0 +1,5 @@ +source: loki +labels: + type: sometype +query: > + {server="demo"} diff --git a/pkg/acquisition/testdata/valid/mock.yaml b/pkg/acquisition/testdata/valid/mock.yaml new file mode 100644 index 00000000000..c84ce4a61b0 --- /dev/null +++ b/pkg/acquisition/testdata/valid/mock.yaml @@ -0,0 +1,4 @@ +source: mock +toto: toto +labels: + type: sometype diff --git a/pkg/acquisition/testdata/valid/s3/minimal.yaml b/pkg/acquisition/testdata/valid/s3/minimal.yaml new file mode 100644 index 00000000000..f83ccc61c05 --- /dev/null +++ b/pkg/acquisition/testdata/valid/s3/minimal.yaml @@ -0,0 +1,4 @@ +source: s3 +labels: + type: sometype +bucket_name: foobar diff --git a/pkg/acquisition/testdata/valid/s3/polling_list.yaml b/pkg/acquisition/testdata/valid/s3/polling_list.yaml new file mode 100644 index 00000000000..ba947672fdf --- /dev/null +++ b/pkg/acquisition/testdata/valid/s3/polling_list.yaml @@ -0,0 +1,5 @@ +source: s3 +labels: + type: sometype +bucket_name: foobar +polling_method: list diff --git a/pkg/acquisition/testdata/valid/s3/polling_sqs.yaml b/pkg/acquisition/testdata/valid/s3/polling_sqs.yaml new file mode 100644 index 00000000000..3491ff0c0cd --- /dev/null +++ b/pkg/acquisition/testdata/valid/s3/polling_sqs.yaml @@ -0,0 +1,5 @@ +source: s3 +labels: + type: sometype +polling_method: sqs +sqs_name: foobar diff --git a/pkg/acquisition/testdata/valid/syslog/minimal.yaml b/pkg/acquisition/testdata/valid/syslog/minimal.yaml new file mode 100644 index 00000000000..f756fbf07bc --- /dev/null +++ b/pkg/acquisition/testdata/valid/syslog/minimal.yaml @@ -0,0 +1,4 @@ +# for syslog, all fields are optional +source: syslog +labels: + type: sometype diff --git a/pkg/acquisition/testdata/valid/victorialogs/full.yaml b/pkg/acquisition/testdata/valid/victorialogs/full.yaml new file mode 100644 index 00000000000..f9e2fc7763f --- /dev/null +++ b/pkg/acquisition/testdata/valid/victorialogs/full.yaml @@ -0,0 +1,10 @@ +source: victorialogs +labels: + type: sometype +url: http://localhost:9428/ +wait_for_ready: 5s +auth: + username: foo + password: bar +query: > + {server="demo"} diff --git a/pkg/acquisition/testdata/valid/victorialogs/minimal.yaml b/pkg/acquisition/testdata/valid/victorialogs/minimal.yaml new file mode 100644 index 00000000000..7593d17e281 --- /dev/null +++ b/pkg/acquisition/testdata/valid/victorialogs/minimal.yaml @@ -0,0 +1,6 @@ +source: victorialogs +labels: + type: sometype +url: http://localhost:9428/ +query: > + {server="demo"} diff --git a/test/bats/01_crowdsec.bats b/test/bats/01_crowdsec.bats index ccc11aed19f..b8c62225d42 100644 --- a/test/bats/01_crowdsec.bats +++ b/test/bats/01_crowdsec.bats @@ -256,7 +256,7 @@ teardown() { config_set '.common.log_media="stdout"' rune -1 wait-for "$CROWDSEC" - assert_stderr --partial "crowdsec init: while loading acquisition config: missing labels in $ACQUIS_DIR/foo.yaml (position 0)" + assert_stderr --partial "crowdsec init: while loading acquisition config: $ACQUIS_DIR/foo.yaml: missing 'source' field" } @test "crowdsec (error if acquisition_path and acquisition_dir are not defined)" { @@ -376,10 +376,10 @@ teardown() { # if filenames are missing, it won't be able to detect source type config_set "$ACQUIS_YAML" '.source="file"' rune -1 wait-for "$CROWDSEC" - assert_stderr --partial "configuring datasource of type file from $ACQUIS_YAML (position 0): no filename or filenames configuration provided" + assert_stderr --partial "crowdsec init: while loading acquisition config: $ACQUIS_YAML: datasource of type file: no filename or filenames configuration provided" config_set "$ACQUIS_YAML" '.filenames=["file.log"]' config_set "$ACQUIS_YAML" '.meh=3' rune -1 wait-for "$CROWDSEC" - assert_stderr --partial "crowdsec init: while loading acquisition config: configuring datasource of type file from $ACQUIS_YAML (position 0): cannot parse FileAcquisition configuration: [6:1] unknown field \"meh\"" + assert_stderr --partial "crowdsec init: while loading acquisition config: $ACQUIS_YAML: datasource of type file: cannot parse FileAcquisition configuration: [6:1] unknown field \"meh\"" } diff --git a/test/bats/appsec.bats b/test/bats/appsec.bats index aee791ec85c..7d63dc3ce15 100644 --- a/test/bats/appsec.bats +++ b/test/bats/appsec.bats @@ -35,7 +35,7 @@ teardown() { EOT rune -1 wait-for "$CROWDSEC" - assert_stderr --partial "crowdsec init: while loading acquisition config: missing labels in $ACQUIS_DIR/appsec.yaml (position 0)" + assert_stderr --partial "crowdsec init: while loading acquisition config: $ACQUIS_DIR/appsec.yaml: missing labels" cat > "$ACQUIS_DIR"/appsec.yaml <<-EOT source: appsec @@ -44,7 +44,7 @@ teardown() { EOT rune -1 wait-for "$CROWDSEC" - assert_stderr --partial "crowdsec init: while loading acquisition config: configuring datasource of type appsec from $ACQUIS_DIR/appsec.yaml (position 0): unable to parse appsec configuration: appsec_config or appsec_config_path must be set" + assert_stderr --partial "crowdsec init: while loading acquisition config: $ACQUIS_DIR/appsec.yaml: datasource of type appsec: unable to parse appsec configuration: appsec_config or appsec_config_path must be set" } @test "appsec allow and ban" { diff --git a/test/bats/crowdsec-acquisition.bats b/test/bats/crowdsec-acquisition.bats index 5874821231b..7c60be84b26 100644 --- a/test/bats/crowdsec-acquisition.bats +++ b/test/bats/crowdsec-acquisition.bats @@ -33,7 +33,7 @@ teardown() { EOT rune -1 "$CROWDSEC" -t - assert_stderr --partial "crowdsec init: while loading acquisition config: configuring datasource of type file from $ACQUIS_DIR/file.yaml (position 0): cannot parse FileAcquisition configuration: [2:1] cannot unmarshal []interface {} into Go struct field Configuration.Filename of type string" + assert_stderr --partial "crowdsec init: while loading acquisition config: $ACQUIS_DIR/file.yaml: datasource of type file: cannot parse FileAcquisition configuration: [2:1] cannot unmarshal []interface {} into Go struct field Configuration.Filename of type string" } @test "empty acquisition file" { @@ -50,7 +50,7 @@ teardown() { EOT rune -1 "$CROWDSEC" -t - assert_stderr --partial "crowdsec init: while loading acquisition config: missing 'source' field in $ACQUIS_DIR/file.yaml (position 0)" + assert_stderr --partial "crowdsec init: while loading acquisition config: $ACQUIS_DIR/file.yaml: missing 'source' field" } @test "malformed acquisition file (duplicate key)" { @@ -62,7 +62,7 @@ teardown() { EOT rune -1 "$CROWDSEC" -t - assert_stderr --partial "crowdsec init: while loading acquisition config: failed to parse $ACQUIS_DIR/file.yaml: position 0: [3:1] mapping key \"filename\" already defined at [1:1]" + assert_stderr --partial "crowdsec init: while loading acquisition config: $ACQUIS_DIR/file.yaml: position 0: [3:1] mapping key \"filename\" already defined at [1:1]" } @test "datasource type detection" { @@ -82,8 +82,8 @@ teardown() { EOT rune -0 "$CROWDSEC" -t - assert_stderr --partial "datasource type missing in $ACQUIS_DIR/file.yaml (position 0): detected 'source=file'" - assert_stderr --partial "datasource type missing in $ACQUIS_DIR/file.yaml (position 1): detected 'source=file'" + assert_stderr --partial "$ACQUIS_DIR/file.yaml (position 0): datasource type missing, detected 'source=file'" + assert_stderr --partial "$ACQUIS_DIR/file.yaml (position 1): datasource type missing, detected 'source=file'" rm -f "$ACQUIS_DIR/file.yaml" @@ -96,7 +96,7 @@ teardown() { rune -0 "$CROWDSEC" -t - assert_stderr --partial "datasource type missing in $ACQUIS_DIR/journal.yaml (position 0): detected 'source=journalctl'" + assert_stderr --partial "$ACQUIS_DIR/journal.yaml: datasource type missing, detected 'source=journalctl'" rm -f "$ACQUIS_DIR/journal.yaml" @@ -113,7 +113,7 @@ teardown() { rune -1 "$CROWDSEC" -t - assert_stderr --partial "crowdsec init: while loading acquisition config: configuring datasource of type docker from $ACQUIS_DIR/bad.yaml (position 0): while parsing DockerAcquisition configuration: [2:1] unknown field \\\"journalctl_filter\\\"" + assert_stderr --partial "crowdsec init: while loading acquisition config: $ACQUIS_DIR/bad.yaml: datasource of type docker: while parsing DockerAcquisition configuration: [2:1] unknown field \\\"journalctl_filter\\\"" } @test "datasource docker (regexp)" { @@ -126,7 +126,7 @@ teardown() { EOT rune -1 "$CROWDSEC" -t - assert_stderr --partial "crowdsec init: while loading acquisition config: configuring datasource of type docker from $ACQUIS_DIR/bad.yaml (position 0): container_name_regexp: error parsing regexp: missing closing ]: \`[abc\`" + assert_stderr --partial "crowdsec init: while loading acquisition config: $ACQUIS_DIR/bad.yaml: datasource of type docker: container_name_regexp: error parsing regexp: missing closing ]: \`[abc\`" } @test "test mode does not fail because of appsec and allowlists" {