diff --git a/internal/generator/vector/api/api.go b/internal/generator/vector/api/api.go new file mode 100644 index 000000000..12b31c0df --- /dev/null +++ b/internal/generator/vector/api/api.go @@ -0,0 +1,6 @@ +package api + +// Api is the API vector configurations +type Api struct { + Enabled bool `json:"enabled,omitempty" yaml:"enabled,omitempty" toml:"enabled,omitempty"` +} diff --git a/internal/generator/vector/api/config.go b/internal/generator/vector/api/config.go index 10a204ed9..785c374f9 100644 --- a/internal/generator/vector/api/config.go +++ b/internal/generator/vector/api/config.go @@ -8,9 +8,21 @@ import ( // Config represents a configuration for vector type Config struct { + Global + + // Api is the set of API keys to values + Api *Api `json:"api,omitempty" yaml:"api,omitempty" toml:"api,omitempty"` + + // Secret is the set of secret ids to secret configurations + Secret map[string]interface{} `json:"secret,omitempty" yaml:"secret,omitempty" toml:"secret,omitempty"` + + // Sources is the set of source ids to source configurations + Sources map[string]interface{} `json:"sources,omitempty" yaml:"sources,omitempty" toml:"sources,omitempty"` // Transforms is the set of transform ids to transform configurations - Transforms map[string]interface{} `json:"transforms" yaml:"transforms" toml:"transforms"` + Transforms map[string]interface{} `json:"transforms,omitempty" yaml:"transforms,omitempty" toml:"transforms,omitempty"` + + Sinks map[string]interface{} `json:"sinks,omitempty" yaml:"sinks,omitempty" toml:"sinks,omitempty"` } // Name is a deprecated method to adapt to the existing generator framework @@ -28,5 +40,8 @@ func (c Config) Template() string { } func (c Config) String() string { - return strings.ReplaceAll(toml.MustMarshal(c), "[transforms]", "") + out := strings.ReplaceAll(toml.MustMarshal(c), "[transforms]", "") + out = strings.ReplaceAll(out, "[sources]", "") + out = strings.ReplaceAll(out, "[sinks]", "") + return out } diff --git a/internal/generator/vector/api/config_test.go b/internal/generator/vector/api/config_test.go new file mode 100644 index 000000000..4e7ba69ae --- /dev/null +++ b/internal/generator/vector/api/config_test.go @@ -0,0 +1,62 @@ +package api_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + vectorapi "github.com/openshift/cluster-logging-operator/internal/generator/vector/api" + "github.com/openshift/cluster-logging-operator/internal/utils/toml" + "github.com/openshift/cluster-logging-operator/test" +) + +const ( + configToml = ` +expire_metrics_secs = 60 +data_dir = "/var/lib/vector/openshift-logging/collector" + +[api] +enabled = true + +# Load sensitive data from files +[secret.kubernetes_secret] +type = "file" +base_path = "/var/run/ocp-collector/secrets" + +[sources.internal_metrics] +type = "internal_metrics" + +[transforms.bar] +type = "remap" + +[sinks.foo] +type = "foo" +` + configYaml = ` +expire_metrics_secs: 60 +data_dir: "/var/lib/vector/openshift-logging/collector" +api: + enabled: true +secret: + kubernetes_secret: + type: "file" + base_path: "/var/run/ocp-collector/secrets" +sources: + internal_metrics: + type: "internal_metrics" +transforms: + bar: + type: "remap" +sinks: + foo: + type: "foo" +` +) + +var _ = Describe("Config", func() { + + It("should highlevel roundtrip serialization without field loss", func() { + config := &vectorapi.Config{} + toml.MustUnMarshal(configToml, config) + Expect(test.YAMLString(config)).To(MatchYAML(configYaml)) + }) + +}) diff --git a/internal/generator/vector/api/global.go b/internal/generator/vector/api/global.go new file mode 100644 index 000000000..03573eeb0 --- /dev/null +++ b/internal/generator/vector/api/global.go @@ -0,0 +1,7 @@ +package api + +// Global are the root level global configuration +type Global struct { + DataDir string `json:"data_dir,omitempty" yaml:"data_dir,omitempty" toml:"data_dir,omitempty"` + ExpireMetricsSec uint `json:"expire_metrics_secs,omitempty" yaml:"expire_metrics_secs,omitempty" toml:"expire_metrics_secs,omitempty"` +} diff --git a/internal/generator/vector/api/sources/file/file.go b/internal/generator/vector/api/sources/file/file.go new file mode 100644 index 000000000..20df4e8cf --- /dev/null +++ b/internal/generator/vector/api/sources/file/file.go @@ -0,0 +1,28 @@ +package file + + +type sourceType string + +const SourceTypeFile sourceType = "file" + +type File struct { + // Type is required to be 'file' + Type sourceType `json:"type" yaml:"type" toml:"type"` + + // Include is file paths to include for this source + Include []string `json:"include" yaml:"include" toml:"include"` + + HostKey string `json:"host_key,omitempty" yaml:"host_key,omitempty" toml:"host_key,omitempty"` + GlobalMinimumCooldownMilliSeconds int64 `json:"glob_minimum_cooldown_ms,omitempty" yaml:"glob_minimum_cooldown_ms" toml:"glob_minimum_cooldown_ms,omitempty"` + IgnoreOlderSecs int64 `json:"ignore_older_secs,omitempty" yaml:"ignore_older_secs,omitempty" toml:"ignore_older_secs,omitempty"` + MaxLineBytes int64 `json:"max_line_bytes,omitempty" yaml:"max_line_bytes,omitempty" toml:"max_line_bytes,omitempty"` + MaxReadBytes int64 `json:"max_read_bytes,omitempty" yaml:"max_read_bytes,omitempty" toml:"max_read_bytes,omitempty"` + RotateWaitSecs int64 `json:"rotate_wait_secs,omitempty" yaml:"rotate_wait_secs,omitempty" toml:"rotate_wait_secs,omitempty"` +} + +func New(include ...string) *File { + return &File{ + Type: SourceTypeFile, + Include: include, + } +} diff --git a/internal/generator/vector/api/suite_test.go b/internal/generator/vector/api/suite_test.go new file mode 100644 index 000000000..9736698f3 --- /dev/null +++ b/internal/generator/vector/api/suite_test.go @@ -0,0 +1,13 @@ +package api_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestSuite(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "[internal][generator][vector][api] Suite") +} diff --git a/internal/generator/vector/api/transforms/logtometric/log_to_metric.go b/internal/generator/vector/api/transforms/logtometric/log_to_metric.go index 2fa995344..309c34472 100644 --- a/internal/generator/vector/api/transforms/logtometric/log_to_metric.go +++ b/internal/generator/vector/api/transforms/logtometric/log_to_metric.go @@ -8,12 +8,12 @@ import ( // LogToMetric is the configuration for the log_to_metric transform type LogToMetric struct { - // Type is required to be 'log_to_metric' - Type string `json:"type" yaml:"type" toml:"type"` - // Inputs is the IDs of the components feeding into this component Inputs []string `json:"inputs" yaml:"inputs" toml:"inputs"` + // Type is required to be 'log_to_metric' + Type string `json:"type" yaml:"type" toml:"type"` + // Metrics is the spec for the Metrics being exposed Metrics []Metric `json:"metrics" yaml:"metrics" toml:"metrics"` } @@ -49,16 +49,16 @@ const ( type Metric struct { Field string `json:"field" yaml:"field" toml:"field"` + Kind MetricsKind `json:"kind,omitempty" yaml:"kind,omitempty" toml:"kind,omitempty"` + MetricName string `json:"name,omitempty" yaml:"name,omitempty" toml:"name,omitempty"` Namespace string `json:"namespace,omitempty" yaml:"namespace,omitempty" toml:"namespace,omitempty"` - Kind MetricsKind `json:"kind,omitempty" yaml:"kind,omitempty" toml:"kind,omitempty"` - - Type MetricsType `json:"type" yaml:"type" toml:"type"` - // Tags optional tags (or labels) to apply to the metric Tags Tags `json:"tags,omitempty" yaml:"tags,omitempty" toml:"tags,omitempty,multiline"` + + Type MetricsType `json:"type" yaml:"type" toml:"type"` } // Tags optional tags (or labels) to apply to the metric diff --git a/internal/generator/vector/api/transforms/remap/remap.go b/internal/generator/vector/api/transforms/remap/remap.go new file mode 100644 index 000000000..38d9d2835 --- /dev/null +++ b/internal/generator/vector/api/transforms/remap/remap.go @@ -0,0 +1,49 @@ +package remap + +import ( + vectorapi "github.com/openshift/cluster-logging-operator/internal/generator/vector/api" +) + +type Remap struct { + id string + // Type is required to be 'log_to_metric' + Type string `json:"type" yaml:"type" toml:"type"` + + // Inputs is the IDs of the components feeding into this component + Inputs []string `json:"inputs" yaml:"inputs" toml:"inputs"` + + // Type is required to be 'log_to_metric' + Source string `json:"source" yaml:"source" toml:"source" multiline:"true" literal:"true"` +} + +func New(id, source string, inputs ...string) *Remap { + return &Remap{ + id: id, + Type: "remap", + Inputs: inputs, + Source: source, + } +} + +// Name is a deprecated method to adapt to the existing generator framework +func (r Remap) Name() string { + return "remap" +} + +// Template is a deprecated method to adapt to the existing generator framework +func (r Remap) Template() string { + return `{{define "` + r.Name() + `" -}} +{{ if ne "" .String }} +{{.}} +{{end}} +{{end}}` +} + +func (r Remap) String() string { + c := vectorapi.Config{ + Transforms: map[string]interface{}{ + r.id: r, + }, + } + return c.String() +} diff --git a/internal/generator/vector/conf/complex.toml b/internal/generator/vector/conf/complex.toml index 4e5fae8c3..c523abfd5 100644 --- a/internal/generator/vector/conf/complex.toml +++ b/internal/generator/vector/conf/complex.toml @@ -12,7 +12,6 @@ path = "/var/run/ocp-collector/secrets" [sources.internal_metrics] type = "internal_metrics" -# Logs from host audit [sources.input_audit_host] type = "file" include = ["/var/log/audit/audit.log"] @@ -20,7 +19,7 @@ host_key = "hostname" glob_minimum_cooldown_ms = 15000 ignore_older_secs = 3600 max_line_bytes = 3145728 -max_read_bytes = 262144 +max_read_bytes = 262144 rotate_wait_secs = 5 [transforms.input_audit_host_meta] @@ -150,7 +149,6 @@ source = ''' ''' -# Logs from kubernetes audit [sources.input_audit_kube] type = "file" include = ["/var/log/kube-apiserver/audit.log"] @@ -158,7 +156,7 @@ host_key = "hostname" glob_minimum_cooldown_ms = 15000 ignore_older_secs = 3600 max_line_bytes = 3145728 -max_read_bytes = 262144 +max_read_bytes = 262144 rotate_wait_secs = 5 [transforms.input_audit_kube_meta] @@ -177,15 +175,14 @@ source = ''' ._internal.openshift.sequence = to_unix_timestamp(now(), unit: "nanoseconds") ''' -# Logs from openshift audit [sources.input_audit_openshift] type = "file" -include = ["/var/log/oauth-apiserver/audit.log","/var/log/openshift-apiserver/audit.log","/var/log/oauth-server/audit.log"] +include = ["/var/log/oauth-apiserver/audit.log", "/var/log/openshift-apiserver/audit.log", "/var/log/oauth-server/audit.log"] host_key = "hostname" glob_minimum_cooldown_ms = 15000 ignore_older_secs = 3600 max_line_bytes = 3145728 -max_read_bytes = 262144 +max_read_bytes = 262144 rotate_wait_secs = 5 [transforms.input_audit_openshift_meta] @@ -204,7 +201,6 @@ source = ''' ._internal.openshift.sequence = to_unix_timestamp(now(), unit: "nanoseconds") ''' -# Logs from ovn audit [sources.input_audit_ovn] type = "file" include = ["/var/log/ovn/acl-audit-log.log"] @@ -212,7 +208,7 @@ host_key = "hostname" glob_minimum_cooldown_ms = 15000 ignore_older_secs = 3600 max_line_bytes = 3145728 -max_read_bytes = 262144 +max_read_bytes = 262144 rotate_wait_secs = 5 [transforms.input_audit_ovn_meta] diff --git a/internal/generator/vector/conf/complex_http_receiver.toml b/internal/generator/vector/conf/complex_http_receiver.toml index 551aedebb..d94587de8 100644 --- a/internal/generator/vector/conf/complex_http_receiver.toml +++ b/internal/generator/vector/conf/complex_http_receiver.toml @@ -12,7 +12,6 @@ path = "/var/run/ocp-collector/secrets" [sources.internal_metrics] type = "internal_metrics" -# Logs from host audit [sources.input_audit_host] type = "file" include = ["/var/log/audit/audit.log"] @@ -20,7 +19,7 @@ host_key = "hostname" glob_minimum_cooldown_ms = 15000 ignore_older_secs = 3600 max_line_bytes = 3145728 -max_read_bytes = 262144 +max_read_bytes = 262144 rotate_wait_secs = 5 [transforms.input_audit_host_meta] @@ -148,7 +147,6 @@ source = ''' } ''' -# Logs from kubernetes audit [sources.input_audit_kube] type = "file" include = ["/var/log/kube-apiserver/audit.log"] @@ -156,7 +154,7 @@ host_key = "hostname" glob_minimum_cooldown_ms = 15000 ignore_older_secs = 3600 max_line_bytes = 3145728 -max_read_bytes = 262144 +max_read_bytes = 262144 rotate_wait_secs = 5 [transforms.input_audit_kube_meta] @@ -175,15 +173,14 @@ source = ''' ._internal.openshift.sequence = to_unix_timestamp(now(), unit: "nanoseconds") ''' -# Logs from openshift audit [sources.input_audit_openshift] type = "file" -include = ["/var/log/oauth-apiserver/audit.log","/var/log/openshift-apiserver/audit.log","/var/log/oauth-server/audit.log"] +include = ["/var/log/oauth-apiserver/audit.log", "/var/log/openshift-apiserver/audit.log", "/var/log/oauth-server/audit.log"] host_key = "hostname" glob_minimum_cooldown_ms = 15000 ignore_older_secs = 3600 max_line_bytes = 3145728 -max_read_bytes = 262144 +max_read_bytes = 262144 rotate_wait_secs = 5 [transforms.input_audit_openshift_meta] @@ -202,7 +199,6 @@ source = ''' ._internal.openshift.sequence = to_unix_timestamp(now(), unit: "nanoseconds") ''' -# Logs from ovn audit [sources.input_audit_ovn] type = "file" include = ["/var/log/ovn/acl-audit-log.log"] @@ -210,7 +206,7 @@ host_key = "hostname" glob_minimum_cooldown_ms = 15000 ignore_older_secs = 3600 max_line_bytes = 3145728 -max_read_bytes = 262144 +max_read_bytes = 262144 rotate_wait_secs = 5 [transforms.input_audit_ovn_meta] diff --git a/internal/generator/vector/filter/openshift/viaq/v1/normalize.go b/internal/generator/vector/filter/openshift/viaq/v1/normalize.go index c9e5f34d5..5eeb2a492 100644 --- a/internal/generator/vector/filter/openshift/viaq/v1/normalize.go +++ b/internal/generator/vector/filter/openshift/viaq/v1/normalize.go @@ -26,7 +26,6 @@ if ._internal.log_source == "syslog" && exists(._internal.structured) {. = merge if ._internal.log_source == "container" && exists(._internal.structured) {.structured = ._internal.structured } ` SetLogLevel = ` - if !exists(._internal.level) { level = null message = ._internal.message @@ -93,31 +92,31 @@ if !exists(._internal.level) { } # attempt 5: Match on the keyword that appears earliest in the message - if level == "default" { - level_patterns = r'(?i)(?emergency|)|(?alert|)|(?critical|)|(?error|)|(?warn(?:ing)?|)|(?notice|)|(?:\b(?info)\b|)|(?debug|)|(?trace|)' - parsed, err = parse_regex(message, level_patterns) - if err == null { - if is_string(parsed.emergency) { - level = "emergency" - } else if is_string(parsed.alert) { - level = "alert" - } else if is_string(parsed.critical) { - level = "critical" - } else if is_string(parsed.error) { - level = "error" - } else if is_string(parsed.warn) { - level = "warn" - } else if is_string(parsed.notice) { - level = "notice" - } else if is_string(parsed.info) { - level = "info" - } else if is_string(parsed.debug) { - level = "debug" - } else if is_string(parsed.trace) { - level = "trace" - } - } - } + if level == "default" { + level_patterns = r'(?i)(?emergency|)|(?alert|)|(?critical|)|(?error|)|(?warn(?:ing)?|)|(?notice|)|(?:\b(?info)\b|)|(?debug|)|(?trace|)' + parsed, err = parse_regex(message, level_patterns) + if err == null { + if is_string(parsed.emergency) { + level = "emergency" + } else if is_string(parsed.alert) { + level = "alert" + } else if is_string(parsed.critical) { + level = "critical" + } else if is_string(parsed.error) { + level = "error" + } else if is_string(parsed.warn) { + level = "warn" + } else if is_string(parsed.notice) { + level = "notice" + } else if is_string(parsed.info) { + level = "info" + } else if is_string(parsed.debug) { + level = "debug" + } else if is_string(parsed.trace) { + level = "trace" + } + } + } } ._internal.level = level } diff --git a/internal/generator/vector/input/audit.toml b/internal/generator/vector/input/audit.toml index 8573fe888..473884cf4 100644 --- a/internal/generator/vector/input/audit.toml +++ b/internal/generator/vector/input/audit.toml @@ -1,4 +1,3 @@ -# Logs from host audit [sources.input_audit_host] type = "file" include = ["/var/log/audit/audit.log"] @@ -6,7 +5,7 @@ host_key = "hostname" glob_minimum_cooldown_ms = 15000 ignore_older_secs = 3600 max_line_bytes = 3145728 -max_read_bytes = 262144 +max_read_bytes = 262144 rotate_wait_secs = 5 [transforms.input_audit_host_meta] @@ -136,7 +135,6 @@ source = ''' ''' -# Logs from kubernetes audit [sources.input_audit_kube] type = "file" include = ["/var/log/kube-apiserver/audit.log"] @@ -144,7 +142,7 @@ host_key = "hostname" glob_minimum_cooldown_ms = 15000 ignore_older_secs = 3600 max_line_bytes = 3145728 -max_read_bytes = 262144 +max_read_bytes = 262144 rotate_wait_secs = 5 [transforms.input_audit_kube_meta] @@ -163,15 +161,14 @@ source = ''' ._internal.openshift.sequence = to_unix_timestamp(now(), unit: "nanoseconds") ''' -# Logs from openshift audit [sources.input_audit_openshift] type = "file" -include = ["/var/log/oauth-apiserver/audit.log","/var/log/openshift-apiserver/audit.log","/var/log/oauth-server/audit.log"] +include = ["/var/log/oauth-apiserver/audit.log", "/var/log/openshift-apiserver/audit.log", "/var/log/oauth-server/audit.log"] host_key = "hostname" glob_minimum_cooldown_ms = 15000 ignore_older_secs = 3600 max_line_bytes = 3145728 -max_read_bytes = 262144 +max_read_bytes = 262144 rotate_wait_secs = 5 [transforms.input_audit_openshift_meta] @@ -190,7 +187,6 @@ source = ''' ._internal.openshift.sequence = to_unix_timestamp(now(), unit: "nanoseconds") ''' -# Logs from ovn audit [sources.input_audit_ovn] type = "file" include = ["/var/log/ovn/acl-audit-log.log"] @@ -198,7 +194,7 @@ host_key = "hostname" glob_minimum_cooldown_ms = 15000 ignore_older_secs = 3600 max_line_bytes = 3145728 -max_read_bytes = 262144 +max_read_bytes = 262144 rotate_wait_secs = 5 [transforms.input_audit_ovn_meta] diff --git a/internal/generator/vector/input/audit_host.toml b/internal/generator/vector/input/audit_host.toml index ca0e30f2a..e152efa14 100644 --- a/internal/generator/vector/input/audit_host.toml +++ b/internal/generator/vector/input/audit_host.toml @@ -1,4 +1,3 @@ -# Logs from host audit [sources.input_myaudit_host] type = "file" include = ["/var/log/audit/audit.log"] @@ -6,7 +5,7 @@ host_key = "hostname" glob_minimum_cooldown_ms = 15000 ignore_older_secs = 3600 max_line_bytes = 3145728 -max_read_bytes = 262144 +max_read_bytes = 262144 rotate_wait_secs = 5 [transforms.input_myaudit_host_meta] diff --git a/internal/generator/vector/input/audit_kube.toml b/internal/generator/vector/input/audit_kube.toml index 2ff461270..04241f9bb 100644 --- a/internal/generator/vector/input/audit_kube.toml +++ b/internal/generator/vector/input/audit_kube.toml @@ -1,4 +1,3 @@ -# Logs from kubernetes audit [sources.input_myaudit_kube] type = "file" include = ["/var/log/kube-apiserver/audit.log"] @@ -6,7 +5,7 @@ host_key = "hostname" glob_minimum_cooldown_ms = 15000 ignore_older_secs = 3600 max_line_bytes = 3145728 -max_read_bytes = 262144 +max_read_bytes = 262144 rotate_wait_secs = 5 [transforms.input_myaudit_kube_meta] diff --git a/internal/generator/vector/input/audit_openshift.toml b/internal/generator/vector/input/audit_openshift.toml index 9867e571c..310e1caeb 100644 --- a/internal/generator/vector/input/audit_openshift.toml +++ b/internal/generator/vector/input/audit_openshift.toml @@ -1,12 +1,11 @@ -# Logs from openshift audit [sources.input_myaudit_openshift] type = "file" -include = ["/var/log/oauth-apiserver/audit.log","/var/log/openshift-apiserver/audit.log","/var/log/oauth-server/audit.log"] +include = ["/var/log/oauth-apiserver/audit.log", "/var/log/openshift-apiserver/audit.log", "/var/log/oauth-server/audit.log"] host_key = "hostname" glob_minimum_cooldown_ms = 15000 ignore_older_secs = 3600 max_line_bytes = 3145728 -max_read_bytes = 262144 +max_read_bytes = 262144 rotate_wait_secs = 5 [transforms.input_myaudit_openshift_meta] diff --git a/internal/generator/vector/input/audit_ovn.toml b/internal/generator/vector/input/audit_ovn.toml index 18eb4776a..2e6a06d02 100644 --- a/internal/generator/vector/input/audit_ovn.toml +++ b/internal/generator/vector/input/audit_ovn.toml @@ -1,4 +1,3 @@ -# Logs from ovn audit [sources.input_myaudit_ovn] type = "file" include = ["/var/log/ovn/acl-audit-log.log"] @@ -6,7 +5,7 @@ host_key = "hostname" glob_minimum_cooldown_ms = 15000 ignore_older_secs = 3600 max_line_bytes = 3145728 -max_read_bytes = 262144 +max_read_bytes = 262144 rotate_wait_secs = 5 [transforms.input_myaudit_ovn_meta] diff --git a/internal/generator/vector/input/internal.go b/internal/generator/vector/input/internal.go index 2c359593f..ff69a0e3a 100644 --- a/internal/generator/vector/input/internal.go +++ b/internal/generator/vector/input/internal.go @@ -3,10 +3,11 @@ package input import ( "fmt" - v1 "github.com/openshift/cluster-logging-operator/internal/generator/vector/filter/openshift/viaq/v1" - "strings" + "github.com/openshift/cluster-logging-operator/internal/generator/vector/api/transforms/remap" + v1 "github.com/openshift/cluster-logging-operator/internal/generator/vector/filter/openshift/viaq/v1" + obs "github.com/openshift/cluster-logging-operator/api/observability/v1" "github.com/openshift/cluster-logging-operator/internal/generator/framework" "github.com/openshift/cluster-logging-operator/internal/generator/vector/elements" @@ -68,21 +69,16 @@ func NewInternalNormalization(id string, logSource, logType interface{}, inputs // Add kubernetes container iostream for all container sources vrls = append(vrls, setKubernetesContainerIOStream) } - vrls = append(vrls, - fmt.Sprintf(fmtLogSource, logSource), + vrls = append(vrls, fmt.Sprintf(fmtLogSource, logSource), logTypeVRL, setHostName, setClusterID, setOpenshiftSequence, v1.SetLogLevel, ) - vrls = append(vrls, addVRLs...) - return elements.Remap{ - ComponentID: id, - Inputs: helpers.MakeInputs(inputs), - VRL: strings.Join(vrls, "\n"), - } + vrl := strings.Join(vrls, "\n") + return remap.New(id, vrl, inputs) } // NewJournalInternalNormalization returns configuration elements to normalize journal log entries to an internal, common data model diff --git a/internal/generator/vector/input/source_test.go b/internal/generator/vector/input/source_test.go index 03325a22f..9973d2783 100644 --- a/internal/generator/vector/input/source_test.go +++ b/internal/generator/vector/input/source_test.go @@ -48,7 +48,7 @@ var _ = Describe("inputs", func() { }, } conf, _ := NewSource(input, constants.OpenshiftNS, *factory.ResourceNames(clf), secrets, framework.NoOptions) - Expect(string(exp)).To(EqualConfigFrom(conf)) + Expect(conf).To(MatchToml(string(exp))) }, Entry("with an application input should generate a container source", obs.InputSpec{ Name: string(obs.InputTypeApplication), diff --git a/internal/generator/vector/source/audit_logs.go b/internal/generator/vector/source/audit_logs.go index 495e8b06e..17348eb17 100644 --- a/internal/generator/vector/source/audit_logs.go +++ b/internal/generator/vector/source/audit_logs.go @@ -1,108 +1,98 @@ package source import ( - "github.com/openshift/cluster-logging-operator/internal/generator/framework" + "github.com/openshift/cluster-logging-operator/internal/generator/vector/api" + sourcesfile "github.com/openshift/cluster-logging-operator/internal/generator/vector/api/sources/file" ) -const HostAuditLogTemplate = ` -{{define "inputSourceHostAuditTemplate" -}} -# {{.Desc}} -[sources.{{.ComponentID}}] -type = "file" -include = ["/var/log/audit/audit.log"] -host_key = "hostname" -glob_minimum_cooldown_ms = 15000 -ignore_older_secs = 3600 -max_line_bytes = 3145728 -max_read_bytes = 262144 -rotate_wait_secs = 5 -{{end}}` - -type HostAuditLog = framework.ConfLiteral - -const OpenshiftAuditLogTemplate = ` -{{define "inputSourceOpenShiftAuditTemplate" -}} -# {{.Desc}} -[sources.{{.ComponentID}}] -type = "file" -include = ["/var/log/oauth-apiserver/audit.log","/var/log/openshift-apiserver/audit.log","/var/log/oauth-server/audit.log"] -host_key = "hostname" -glob_minimum_cooldown_ms = 15000 -ignore_older_secs = 3600 -max_line_bytes = 3145728 -max_read_bytes = 262144 -rotate_wait_secs = 5 -{{end}} -` - -type OpenshiftAuditLog = framework.ConfLiteral - -const K8sAuditLogTemplate = ` -{{define "inputSourceK8sAuditTemplate" -}} -# {{.Desc}} -[sources.{{.ComponentID}}] -type = "file" -include = ["/var/log/kube-apiserver/audit.log"] -host_key = "hostname" -glob_minimum_cooldown_ms = 15000 -ignore_older_secs = 3600 -max_line_bytes = 3145728 -max_read_bytes = 262144 -rotate_wait_secs = 5 -{{end}} -` - -type OVNAuditLog = framework.ConfLiteral - -const OVNAuditLogTemplate = ` -{{define "inputSourceOVNAuditTemplate" -}} -# {{.Desc}} -[sources.{{.ComponentID}}] -type = "file" -include = ["/var/log/ovn/acl-audit-log.log"] -host_key = "hostname" -glob_minimum_cooldown_ms = 15000 -ignore_older_secs = 3600 -max_line_bytes = 3145728 -max_read_bytes = 262144 -rotate_wait_secs = 5 -{{end}} -` +const ( + GlobalMinimumCooldown = 15000 + IgnoreOlderSecs = 3600 + MaxLineBytes = 3145728 + MaxReadBytes = 262144 + RotateWaitSecs = 5 +) -type K8sAuditLog = framework.ConfLiteral +type HostAuditLog struct { + api.Config +} func NewHostAuditLog(id string) HostAuditLog { + f := sourcesfile.New("/var/log/audit/audit.log") + f.HostKey = "hostname" + f.GlobalMinimumCooldownMilliSeconds = GlobalMinimumCooldown + f.IgnoreOlderSecs = IgnoreOlderSecs + f.MaxLineBytes = MaxLineBytes + f.MaxReadBytes = MaxReadBytes + f.RotateWaitSecs = RotateWaitSecs return HostAuditLog{ - ComponentID: id, - Desc: "Logs from host audit", - TemplateName: "inputSourceHostAuditTemplate", - TemplateStr: HostAuditLogTemplate, + Config: api.Config{ + Sources: map[string]interface{}{ + id: f, + }, + }, } } -func NewK8sAuditLog(id string) K8sAuditLog { - return K8sAuditLog{ - ComponentID: id, - Desc: "Logs from kubernetes audit", - TemplateName: "inputSourceK8sAuditTemplate", - TemplateStr: K8sAuditLogTemplate, - } +type OpenshiftAuditLog struct { + api.Config } func NewOpenshiftAuditLog(id string) OpenshiftAuditLog { + f := sourcesfile.New("/var/log/oauth-apiserver/audit.log", "/var/log/openshift-apiserver/audit.log", "/var/log/oauth-server/audit.log") + f.HostKey = "hostname" + f.GlobalMinimumCooldownMilliSeconds = GlobalMinimumCooldown + f.IgnoreOlderSecs = IgnoreOlderSecs + f.MaxLineBytes = MaxLineBytes + f.MaxReadBytes = MaxReadBytes + f.RotateWaitSecs = RotateWaitSecs return OpenshiftAuditLog{ - ComponentID: id, - Desc: "Logs from openshift audit", - TemplateName: "inputSourceOpenShiftAuditTemplate", - TemplateStr: OpenshiftAuditLogTemplate, + Config: api.Config{ + Sources: map[string]interface{}{ + id: f, + }, + }, } } -func NewOVNAuditLog(id string) OVNAuditLog { - return OVNAuditLog{ - ComponentID: id, - Desc: "Logs from ovn audit", - TemplateName: "inputSourceOVNAuditTemplate", - TemplateStr: OVNAuditLogTemplate, +type K8sAuditLog struct { + api.Config +} + +func NewK8sAuditLog(id string) K8sAuditLog { + f := sourcesfile.New("/var/log/kube-apiserver/audit.log") + f.HostKey = "hostname" + f.GlobalMinimumCooldownMilliSeconds = GlobalMinimumCooldown + f.IgnoreOlderSecs = IgnoreOlderSecs + f.MaxLineBytes = MaxLineBytes + f.MaxReadBytes = MaxReadBytes + f.RotateWaitSecs = RotateWaitSecs + return K8sAuditLog{ + Config: api.Config{ + Sources: map[string]interface{}{ + id: f, + }, + }, + } +} + +type OVNAuditLog struct { + api.Config +} + +func NewOVNAuditLog(id string) K8sAuditLog { + f := sourcesfile.New("/var/log/ovn/acl-audit-log.log") + f.HostKey = "hostname" + f.GlobalMinimumCooldownMilliSeconds = GlobalMinimumCooldown + f.IgnoreOlderSecs = IgnoreOlderSecs + f.MaxLineBytes = MaxLineBytes + f.MaxReadBytes = MaxReadBytes + f.RotateWaitSecs = RotateWaitSecs + return K8sAuditLog{ + Config: api.Config{ + Sources: map[string]interface{}{ + id: f, + }, + }, } } diff --git a/internal/utils/toml/toml.go b/internal/utils/toml/toml.go index 2bb9718b3..82fb98b08 100644 --- a/internal/utils/toml/toml.go +++ b/internal/utils/toml/toml.go @@ -2,15 +2,22 @@ package toml import ( "bytes" + "fmt" toml "github.com/pelletier/go-toml" ) func MustMarshal(v interface{}) string { out := new(bytes.Buffer) - encoder := toml.NewEncoder(out).Indentation("") + encoder := toml.NewEncoder(out).Indentation("").Order(toml.OrderPreserve) if err := encoder.Encode(v); err != nil { panic(err) } return out.String() } + +func MustUnMarshal(s string, v interface{}) { + if err := toml.NewDecoder(bytes.NewBufferString(s)).Decode(v); err != nil { + panic(fmt.Sprintf("Error unmarshalling toml: %v\n%s\n", err, s)) + } +} diff --git a/test/matchers/generator_element.go b/test/matchers/generator_element.go index 63c64780a..bbf60f205 100644 --- a/test/matchers/generator_element.go +++ b/test/matchers/generator_element.go @@ -23,8 +23,8 @@ func EqualConfigFrom(actual interface{}) types.GomegaMatcher { } func (m *GeneratorElementMatcher) Match(expected interface{}) (bool, error) { - if expected := reflect.TypeOf(expected); expected.Kind() != reflect.String { - return false, fmt.Errorf("The 'actual' type is expected to be a string but is: %s", expected.Name()) + if expectedType := reflect.TypeOf(expected); expectedType.Kind() != reflect.String { + return false, fmt.Errorf("The 'actual' type is expected to be a string but is: %s", expectedType.Name()) } conf, err := generateConf(m.actual) m.err = err @@ -70,8 +70,10 @@ func generateConf(expected interface{}) (string, error) { } else { return "", fmt.Errorf("Matcher unable to cast 'expected' type %q to a generator.Element", expType.Name()) } + case expType.Kind() == reflect.String: + return expected.(string), nil default: - return "", fmt.Errorf("Matcher does not support 'expected' kind %q or element type: %q", expType.Kind(), expType.Name()) + return "", fmt.Errorf("Matcher does not support 'expected' kind %q or element type: %q, expected: %v", expType.Kind(), expType.Name(), expected) } g := framework.MakeGenerator() return g.GenerateConf(els...) diff --git a/test/matchers/match_toml.go b/test/matchers/match_toml.go new file mode 100644 index 000000000..35470072a --- /dev/null +++ b/test/matchers/match_toml.go @@ -0,0 +1,48 @@ +package matchers + +import ( + "fmt" + "strings" + + "github.com/google/go-cmp/cmp" + "github.com/onsi/gomega/types" +) + +type TomlMatcher struct { + actualToml string + expToml interface{} + diff string + err error +} + +func MatchToml(expected interface{}) types.GomegaMatcher { + return &TomlMatcher{ + expToml: expected, + } +} + +func (m *TomlMatcher) Match(actual interface{}) (bool, error) { + m.actualToml, m.err = generateConf(actual) + if m.err != nil { + return false, m.err + } + actualConfig := strings.Join(normalize(m.actualToml, true), "\n") + expConfig := strings.Join(normalize(m.expToml.(string), true), "\n") + + m.diff = cmp.Diff(expConfig, actualConfig) + return m.diff == "", nil +} + +func (m *TomlMatcher) FailureMessage(expected interface{}) (message string) { + if m.err != nil { + return fmt.Sprintf("Error generating 'expected' conf: %v", m.err) + } + return fmt.Sprintf("Expected element to produce a config from 'elements'\nexpected:\n>>>\n%s\n\n<<<\n\n\n\nactual\n>>>:\n\n%s\n\n<<<\ndiff: %s\n", expected, m.actualToml, m.diff) +} + +func (m *TomlMatcher) NegatedFailureMessage(expected interface{}) (message string) { + if m.err != nil { + return fmt.Sprintf("Error generating 'expected' conf: %v", m.err) + } + return fmt.Sprintf("Expected element to not produce a config from 'elements'\nexpected:\n%s\n\nactual:\n\n%s\n\ndiff: %s\n", expected, m.actualToml, m.diff) +}