NETOBSERV-1935: enable metrics from list/nested fields

Author: jotak

apis/flowmetrics/v1alpha1/flowmetric_types.go

@@ -93,6 +93,11 @@ type FlowMetricSpec struct {
 	// +optional
 	Labels []string `json:"labels"`
 
+	// `flatten` is a list of list-type fields that must be flattened, such as Interfaces and NetworkEvents. Flattened fields generate one metric per item in that field.
+	// For instance, when flattening `Interfaces` on a bytes counter, a flow having Interfaces [br-ex, ens5] increases one counter for `br-ex` and another for `ens5`.
+	// +optional
+	Flatten []string `json:"flatten"`
+
 	// Set the `remap` property to use different names for the generated metric labels than the flow fields. Use the origin flow fields as keys, and the desired label names as values.
 	// +optional
 	Remap map[string]string `json:"remap"`

apis/flowmetrics/v1alpha1/flowmetric_webhook.go

@@ -100,12 +100,13 @@ func validateFlowMetric(_ context.Context, fMetric *FlowMetric) (admission.Warni
 				fmt.Sprintf("invalid label name: %s", fMetric.Spec.Labels)))
 		}
 
+		labelsMap := make(map[string]any, len(fMetric.Spec.Labels))
+		for _, label := range fMetric.Spec.Labels {
+			labelsMap[label] = nil
+		}
+
 		// Only fields defined as Labels are valid for remapping
 		if len(fMetric.Spec.Remap) != 0 {
-			labelsMap := make(map[string]any, len(fMetric.Spec.Labels))
-			for _, label := range fMetric.Spec.Labels {
-				labelsMap[label] = nil
-			}
 			var invalidMapping []string
 			for toRemap := range fMetric.Spec.Remap {
 				if _, ok := labelsMap[toRemap]; !ok {
@@ -117,6 +118,20 @@ func validateFlowMetric(_ context.Context, fMetric *FlowMetric) (admission.Warni
 					fmt.Sprintf("some fields defined for remapping are not defined as labels: %v", invalidMapping)))
 			}
 		}
+
+		// Only fields defined as Labels are valid for flattening
+		if len(fMetric.Spec.Flatten) != 0 {
+			var invalidFlatten []string
+			for _, toFlatten := range fMetric.Spec.Flatten {
+				if _, ok := labelsMap[toFlatten]; !ok {
+					invalidFlatten = append(invalidFlatten, toFlatten)
+				}
+			}
+			if len(invalidFlatten) > 0 {
+				allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "flatten"), fMetric.Spec.Flatten,
+					fmt.Sprintf("some fields defined for flattening are not defined as labels: %v", invalidFlatten)))
+			}
+		}
 	}
 
 	if fMetric.Spec.ValueField != "" {

apis/flowmetrics/v1alpha1/zz_generated.deepcopy.go

@@ -196,6 +196,13 @@ spec:
                   - matchType
                   type: object
                 type: array
+              flatten:
+                description: |-
+                  `flatten` is a list of list-type fields that must be flattened, such as Interfaces and NetworkEvents. Flattened fields generate one metric per item in that field.
+                  For instance, when flattening `Interfaces` on a bytes counter, a flow having Interfaces [br-ex, ens5] increases one counter for `br-ex` and another for `ens5`.
+                items:
+                  type: string
+                type: array
               labels:
                 description: |-
                   `labels` is a list of fields that should be used as Prometheus labels, also known as dimensions.

bundle/manifests/flows.netobserv.io_flowmetrics.yaml

@@ -186,6 +186,13 @@ spec:
                   - matchType
                   type: object
                 type: array
+              flatten:
+                description: |-
+                  `flatten` is a list of list-type fields that must be flattened, such as Interfaces and NetworkEvents. Flattened fields generate one metric per item in that field.
+                  For instance, when flattening `Interfaces` on a bytes counter, a flow having Interfaces [br-ex, ens5] increases one counter for `br-ex` and another for `ens5`.
+                items:
+                  type: string
+                type: array
               labels:
                 description: |-
                   `labels` is a list of fields that should be used as Prometheus labels, also known as dimensions.

config/crd/bases/flows.netobserv.io_flowmetrics.yaml

@@ -0,0 +1,14 @@
+# Count flows per network interface
+# More examples in https://github.com/netobserv/network-observability-operator/tree/main/config/samples/flowmetrics
+apiVersion: flows.netobserv.io/v1alpha1
+kind: FlowMetric
+metadata:
+  name: per-interface
+  namespace: netobserv
+spec:
+  metricName: per_interface
+  type: Counter
+  labels: [Interfaces]
+  flatten: [Interfaces]
+  remap:
+    Interfaces: interface

config/samples/flowmetrics/per-interface.yaml

@@ -270,6 +270,7 @@ func flowMetricToFLP(flowMetric *metricslatest.FlowMetricSpec) (*api.MetricsItem
 		Filters:  []api.MetricsFilter{},
 		Labels:   flowMetric.Labels,
 		Remap:    flowMetric.Remap,
+		Flatten:  flowMetric.Flatten,
 		ValueKey: flowMetric.ValueField,
 	}
 	for _, f := range metrics.GetFilters(flowMetric) {

controllers/flp/flp_pipeline_builder.go

@@ -151,6 +151,14 @@ be used to eliminate duplicates: `Duplicate != "true"` and `FlowDirection = "0"`
 Refer to the documentation for the list of available fields: https://docs.openshift.com/container-platform/latest/observability/network_observability/json-flows-format-reference.html.<br/>
         </td>
         <td>false</td>
+      </tr><tr>
+        <td><b>flatten</b></td>
+        <td>[]string</td>
+        <td>
+          `flatten` is a list of list-type fields that must be flattened, such as Interfaces and NetworkEvents. Flattened fields generate one metric per item in that field.
+For instance, when flattening `Interfaces` on a bytes counter, a flow having Interfaces [br-ex, ens5] increases one counter for `br-ex` and another for `ens5`.<br/>
+        </td>
+        <td>false</td>
       </tr><tr>
         <td><b>labels</b></td>
         <td>[]string</td>

docs/FlowMetric.md

@@ -88,3 +88,5 @@ require (
 )
 
 replace github.com/prometheus/common v0.55.0 => github.com/netobserv/prometheus-common v0.55.0-netobserv
+
+replace github.com/netobserv/flowlogs-pipeline => github.com/jotak/flowlogs-pipeline v0.0.0-20241113134338-68583d85dd5e

go.mod

@@ -56,6 +56,8 @@ github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM=
 github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/jotak/flowlogs-pipeline v0.0.0-20241113134338-68583d85dd5e h1:+Yn7FfcXQPdb99vl13e8Tppyeq0snWMz+zJH/Q2NMX8=
+github.com/jotak/flowlogs-pipeline v0.0.0-20241113134338-68583d85dd5e/go.mod h1:wnCpWttAFkLSSxOcfCkd9zA5pwV/1OcxS5tAfAxNWEc=
 github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
 github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
@@ -88,8 +90,6 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
-github.com/netobserv/flowlogs-pipeline v1.6.1-crc0.0.20240920063618-2ea1a7ce77b8 h1:Wumfysno927/yCK8RAG/1AtTDpQQzSn9PkoGBLu2Khk=
-github.com/netobserv/flowlogs-pipeline v1.6.1-crc0.0.20240920063618-2ea1a7ce77b8/go.mod h1:VVHQK3sEL91gl6CQkrcE+RAi9vMat5gaVaZLqPDoKlQ=
 github.com/netobserv/prometheus-common v0.55.0-netobserv h1:Fapr74g0S3gRh/kTTyv9Ytm4DJJfFuUTEToiU/np9eg=
 github.com/netobserv/prometheus-common v0.55.0-netobserv/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8=
 github.com/onsi/ginkgo/v2 v2.20.2 h1:7NVCeyIWROIAheY21RLS+3j2bb52W0W82tkberYytp4=