LOG-6385: Refactor audit input generation to use struct

internal/generator/vector/api/api.go

@@ -0,0 +1,6 @@
+package api
+
+// Api is the API vector configurations
+type Api struct {
+	Enabled bool `json:"enabled,omitempty" yaml:"enabled,omitempty" toml:"enabled,omitempty"`
+}

internal/generator/vector/api/config.go

@@ -8,9 +8,21 @@ import (
 
 // Config represents a configuration for vector
 type Config struct {
+	Global
+
+	// Api is the set of API keys to values
+	Api *Api `json:"api,omitempty" yaml:"api,omitempty" toml:"api,omitempty"`
+
+	// Secret is the set of secret ids to secret configurations
+	Secret map[string]interface{} `json:"secret,omitempty" yaml:"secret,omitempty" toml:"secret,omitempty"`
+
+	// Sources is the set of source ids to source configurations
+	Sources map[string]interface{} `json:"sources,omitempty" yaml:"sources,omitempty" toml:"sources,omitempty"`
 
 	// Transforms is the set of transform ids to transform configurations
-	Transforms map[string]interface{} `json:"transforms" yaml:"transforms" toml:"transforms"`
+	Transforms map[string]interface{} `json:"transforms,omitempty" yaml:"transforms,omitempty" toml:"transforms,omitempty"`
+
+	Sinks map[string]interface{} `json:"sinks,omitempty" yaml:"sinks,omitempty" toml:"sinks,omitempty"`
 }
 
 // Name is a deprecated method to adapt to the existing generator framework
@@ -28,5 +40,8 @@ func (c Config) Template() string {
 }
 
 func (c Config) String() string {
-	return strings.ReplaceAll(toml.MustMarshal(c), "[transforms]", "")
+	out := strings.ReplaceAll(toml.MustMarshal(c), "[transforms]", "")
+	out = strings.ReplaceAll(out, "[sources]", "")
+	out = strings.ReplaceAll(out, "[sinks]", "")
+	return out
 }

internal/generator/vector/api/config_test.go

@@ -0,0 +1,62 @@
+package api_test
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	vectorapi "github.com/openshift/cluster-logging-operator/internal/generator/vector/api"
+	"github.com/openshift/cluster-logging-operator/internal/utils/toml"
+	"github.com/openshift/cluster-logging-operator/test"
+)
+
+const (
+	configToml = `
+expire_metrics_secs = 60
+data_dir = "/var/lib/vector/openshift-logging/collector"
+
+[api]
+enabled = true
+
+# Load sensitive data from files
+[secret.kubernetes_secret]
+type = "file"
+base_path = "/var/run/ocp-collector/secrets"
+
+[sources.internal_metrics]
+type = "internal_metrics"
+
+[transforms.bar]
+type = "remap"
+
+[sinks.foo]
+type = "foo"
+`
+	configYaml = `
+expire_metrics_secs: 60
+data_dir: "/var/lib/vector/openshift-logging/collector"
+api:
+  enabled: true
+secret:
+  kubernetes_secret:
+    type: "file"
+    base_path: "/var/run/ocp-collector/secrets"
+sources:
+  internal_metrics:
+    type: "internal_metrics"
+transforms:
+  bar:
+    type: "remap"
+sinks:
+  foo:
+    type: "foo"
+`
+)
+
+var _ = Describe("Config", func() {
+
+	It("should highlevel roundtrip serialization without field loss", func() {
+		config := &vectorapi.Config{}
+		toml.MustUnMarshal(configToml, config)
+		Expect(test.YAMLString(config)).To(MatchYAML(configYaml))
+	})
+
+})

internal/generator/vector/api/global.go

@@ -0,0 +1,7 @@
+package api
+
+// Global are the root level global configuration
+type Global struct {
+	DataDir          string `json:"data_dir,omitempty" yaml:"data_dir,omitempty" toml:"data_dir,omitempty"`
+	ExpireMetricsSec uint   `json:"expire_metrics_secs,omitempty" yaml:"expire_metrics_secs,omitempty" toml:"expire_metrics_secs,omitempty"`
+}

internal/generator/vector/api/sources/file/file.go

@@ -0,0 +1,28 @@
+package file
+
+
+type sourceType string
+
+const SourceTypeFile sourceType = "file"
+
+type File struct {
+	// Type is required to be 'file'
+	Type sourceType `json:"type" yaml:"type" toml:"type"`
+
+	// Include is file paths to include for this source
+	Include []string `json:"include" yaml:"include" toml:"include"`
+
+	HostKey                           string `json:"host_key,omitempty" yaml:"host_key,omitempty" toml:"host_key,omitempty"`
+	GlobalMinimumCooldownMilliSeconds int64  `json:"glob_minimum_cooldown_ms,omitempty" yaml:"glob_minimum_cooldown_ms" toml:"glob_minimum_cooldown_ms,omitempty"`
+	IgnoreOlderSecs                   int64  `json:"ignore_older_secs,omitempty" yaml:"ignore_older_secs,omitempty" toml:"ignore_older_secs,omitempty"`
+	MaxLineBytes                      int64  `json:"max_line_bytes,omitempty" yaml:"max_line_bytes,omitempty" toml:"max_line_bytes,omitempty"`
+	MaxReadBytes                      int64  `json:"max_read_bytes,omitempty" yaml:"max_read_bytes,omitempty" toml:"max_read_bytes,omitempty"`
+	RotateWaitSecs                    int64  `json:"rotate_wait_secs,omitempty" yaml:"rotate_wait_secs,omitempty" toml:"rotate_wait_secs,omitempty"`
+}
+
+func New(include ...string) *File {
+	return &File{
+		Type:    SourceTypeFile,
+		Include: include,
+	}
+}

internal/generator/vector/api/suite_test.go

@@ -0,0 +1,13 @@
+package api_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func TestSuite(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "[internal][generator][vector][api] Suite")
+}

internal/generator/vector/api/transforms/logtometric/log_to_metric.go

@@ -8,12 +8,12 @@ import (
 
 // LogToMetric is the configuration for the log_to_metric transform
 type LogToMetric struct {
-	// Type is required to be 'log_to_metric'
-	Type string `json:"type" yaml:"type" toml:"type"`
-
 	// Inputs is the IDs of the components feeding into this component
 	Inputs []string `json:"inputs" yaml:"inputs" toml:"inputs"`
 
+	// Type is required to be 'log_to_metric'
+	Type string `json:"type" yaml:"type" toml:"type"`
+
 	// Metrics is the spec for the Metrics being exposed
 	Metrics []Metric `json:"metrics" yaml:"metrics" toml:"metrics"`
 }
@@ -49,16 +49,16 @@ const (
 type Metric struct {
 	Field string `json:"field" yaml:"field" toml:"field"`
 
+	Kind MetricsKind `json:"kind,omitempty" yaml:"kind,omitempty" toml:"kind,omitempty"`
+
 	MetricName string `json:"name,omitempty" yaml:"name,omitempty" toml:"name,omitempty"`
 
 	Namespace string `json:"namespace,omitempty" yaml:"namespace,omitempty" toml:"namespace,omitempty"`
 
-	Kind MetricsKind `json:"kind,omitempty" yaml:"kind,omitempty" toml:"kind,omitempty"`
-
-	Type MetricsType `json:"type" yaml:"type" toml:"type"`
-
 	// Tags optional tags (or labels) to apply to the metric
 	Tags Tags `json:"tags,omitempty" yaml:"tags,omitempty"  toml:"tags,omitempty,multiline"`
+
+	Type MetricsType `json:"type" yaml:"type" toml:"type"`
 }
 
 // Tags optional tags (or labels) to apply to the metric

internal/generator/vector/api/transforms/remap/remap.go

@@ -0,0 +1,49 @@
+package remap
+
+import (
+	vectorapi "github.com/openshift/cluster-logging-operator/internal/generator/vector/api"
+)
+
+type Remap struct {
+	id string
+	// Type is required to be 'log_to_metric'
+	Type string `json:"type" yaml:"type" toml:"type"`
+
+	// Inputs is the IDs of the components feeding into this component
+	Inputs []string `json:"inputs" yaml:"inputs" toml:"inputs"`
+
+	// Type is required to be 'log_to_metric'
+	Source string `json:"source" yaml:"source" toml:"source" multiline:"true" literal:"true"`
+}
+
+func New(id, source string, inputs ...string) *Remap {
+	return &Remap{
+		id:     id,
+		Type:   "remap",
+		Inputs: inputs,
+		Source: source,
+	}
+}
+
+// Name is a deprecated method to adapt to the existing generator framework
+func (r Remap) Name() string {
+	return "remap"
+}
+
+// Template is a deprecated method to adapt to the existing generator framework
+func (r Remap) Template() string {
+	return `{{define "` + r.Name() + `" -}}
+{{ if ne "" .String }}
+{{.}}
+{{end}}
+{{end}}`
+}
+
+func (r Remap) String() string {
+	c := vectorapi.Config{
+		Transforms: map[string]interface{}{
+			r.id: r,
+		},
+	}
+	return c.String()
+}

internal/generator/vector/conf/complex.toml

@@ -12,15 +12,14 @@ path = "/var/run/ocp-collector/secrets"
 [sources.internal_metrics]
 type = "internal_metrics"
 
-# Logs from host audit
 [sources.input_audit_host]
 type = "file"
 include = ["/var/log/audit/audit.log"]
 host_key = "hostname"
 glob_minimum_cooldown_ms = 15000
 ignore_older_secs = 3600
 max_line_bytes = 3145728
-max_read_bytes =  262144
+max_read_bytes = 262144
 rotate_wait_secs = 5
 
 [transforms.input_audit_host_meta]
@@ -150,15 +149,14 @@ source = '''
 
 '''
 
-# Logs from kubernetes audit
 [sources.input_audit_kube]
 type = "file"
 include = ["/var/log/kube-apiserver/audit.log"]
 host_key = "hostname"
 glob_minimum_cooldown_ms = 15000
 ignore_older_secs = 3600
 max_line_bytes = 3145728
-max_read_bytes =  262144
+max_read_bytes = 262144
 rotate_wait_secs = 5
 
 [transforms.input_audit_kube_meta]
@@ -177,15 +175,14 @@ source = '''
   ._internal.openshift.sequence = to_unix_timestamp(now(), unit: "nanoseconds")
 '''
 
-# Logs from openshift audit
 [sources.input_audit_openshift]
 type = "file"
-include = ["/var/log/oauth-apiserver/audit.log","/var/log/openshift-apiserver/audit.log","/var/log/oauth-server/audit.log"]
+include = ["/var/log/oauth-apiserver/audit.log", "/var/log/openshift-apiserver/audit.log", "/var/log/oauth-server/audit.log"]
 host_key = "hostname"
 glob_minimum_cooldown_ms = 15000
 ignore_older_secs = 3600
 max_line_bytes = 3145728
-max_read_bytes =  262144
+max_read_bytes = 262144
 rotate_wait_secs = 5
 
 [transforms.input_audit_openshift_meta]
@@ -204,15 +201,14 @@ source = '''
   ._internal.openshift.sequence = to_unix_timestamp(now(), unit: "nanoseconds")
 '''
 
-# Logs from ovn audit
 [sources.input_audit_ovn]
 type = "file"
 include = ["/var/log/ovn/acl-audit-log.log"]
 host_key = "hostname"
 glob_minimum_cooldown_ms = 15000
 ignore_older_secs = 3600
 max_line_bytes = 3145728
-max_read_bytes =  262144
+max_read_bytes = 262144
 rotate_wait_secs = 5
 
 [transforms.input_audit_ovn_meta]

internal/generator/vector/conf/complex_http_receiver.toml

@@ -12,15 +12,14 @@ path = "/var/run/ocp-collector/secrets"
 [sources.internal_metrics]
 type = "internal_metrics"
 
-# Logs from host audit
 [sources.input_audit_host]
 type = "file"
 include = ["/var/log/audit/audit.log"]
 host_key = "hostname"
 glob_minimum_cooldown_ms = 15000
 ignore_older_secs = 3600
 max_line_bytes = 3145728
-max_read_bytes =  262144
+max_read_bytes = 262144
 rotate_wait_secs = 5
 
 [transforms.input_audit_host_meta]
@@ -148,15 +147,14 @@ source = '''
   }
 '''
 
-# Logs from kubernetes audit
 [sources.input_audit_kube]
 type = "file"
 include = ["/var/log/kube-apiserver/audit.log"]
 host_key = "hostname"
 glob_minimum_cooldown_ms = 15000
 ignore_older_secs = 3600
 max_line_bytes = 3145728
-max_read_bytes =  262144
+max_read_bytes = 262144
 rotate_wait_secs = 5
 
 [transforms.input_audit_kube_meta]
@@ -175,15 +173,14 @@ source = '''
   ._internal.openshift.sequence = to_unix_timestamp(now(), unit: "nanoseconds")
 '''
 
-# Logs from openshift audit
 [sources.input_audit_openshift]
 type = "file"
-include = ["/var/log/oauth-apiserver/audit.log","/var/log/openshift-apiserver/audit.log","/var/log/oauth-server/audit.log"]
+include = ["/var/log/oauth-apiserver/audit.log", "/var/log/openshift-apiserver/audit.log", "/var/log/oauth-server/audit.log"]
 host_key = "hostname"
 glob_minimum_cooldown_ms = 15000
 ignore_older_secs = 3600
 max_line_bytes = 3145728
-max_read_bytes =  262144
+max_read_bytes = 262144
 rotate_wait_secs = 5
 
 [transforms.input_audit_openshift_meta]
@@ -202,15 +199,14 @@ source = '''
   ._internal.openshift.sequence = to_unix_timestamp(now(), unit: "nanoseconds")
 '''
 
-# Logs from ovn audit
 [sources.input_audit_ovn]
 type = "file"
 include = ["/var/log/ovn/acl-audit-log.log"]
 host_key = "hostname"
 glob_minimum_cooldown_ms = 15000
 ignore_older_secs = 3600
 max_line_bytes = 3145728
-max_read_bytes =  262144
+max_read_bytes = 262144
 rotate_wait_secs = 5
 
 [transforms.input_audit_ovn_meta]