Skip to content

Commit ca8337c

Browse files
jotakjotak
authored
NETOBSERV-1935: enable metrics from list/nested fields (#863)
* NETOBSERV-1935: enable metrics from list/nested fields * use merge flp deps * Fix webhook validation for nested fields * Add netpol events as predefined metrics 3 new metrics: - node_network_policy_events_total - namespace_network_policy_events_total (enabled by default) - workload_network_policy_events_total And their related charts * NetEvents: use filter name consistent with column
1 parent 90c23ce commit ca8337c

22 files changed

+215
-19
lines changed

‎apis/flowcollector/v1beta1/flowcollector_webhook_test.go‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,14 @@ func TestBeta1ConversionRoundtrip_Metrics(t *testing.T) {
123123
err := initial.ConvertTo(&converted)
124124
assert.NoError(err)
125125

126-
expectedDefaultMetrics := []v1beta2.FLPMetric{"namespace_egress_packets_total", "namespace_flows_total", "namespace_rtt_seconds", "namespace_drop_packets_total", "namespace_dns_latency_seconds"}
126+
expectedDefaultMetrics := []v1beta2.FLPMetric{
127+
"namespace_egress_packets_total",
128+
"namespace_flows_total",
129+
"namespace_rtt_seconds",
130+
"namespace_drop_packets_total",
131+
"namespace_dns_latency_seconds",
132+
"namespace_network_policy_events_total",
133+
}
127134
assert.Equal([]v1beta2.FLPAlert{v1beta2.AlertLokiError}, converted.Spec.Processor.Metrics.DisableAlerts)
128135
assert.NotNil(converted.Spec.Processor.Metrics.IncludeList)
129136
assert.Equal(expectedDefaultMetrics, *converted.Spec.Processor.Metrics.IncludeList)

‎apis/flowcollector/v1beta2/flowcollector_types.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -531,7 +531,7 @@ const (
531531
)
532532

533533
// Metric name. More information in https://github.com/netobserv/network-observability-operator/blob/main/docs/Metrics.md.
534-
// +kubebuilder:validation:Enum:="namespace_egress_bytes_total";"namespace_egress_packets_total";"namespace_ingress_bytes_total";"namespace_ingress_packets_total";"namespace_flows_total";"node_egress_bytes_total";"node_egress_packets_total";"node_ingress_bytes_total";"node_ingress_packets_total";"node_flows_total";"workload_egress_bytes_total";"workload_egress_packets_total";"workload_ingress_bytes_total";"workload_ingress_packets_total";"workload_flows_total";"namespace_drop_bytes_total";"namespace_drop_packets_total";"node_drop_bytes_total";"node_drop_packets_total";"workload_drop_bytes_total";"workload_drop_packets_total";"namespace_rtt_seconds";"node_rtt_seconds";"workload_rtt_seconds";"namespace_dns_latency_seconds";"node_dns_latency_seconds";"workload_dns_latency_seconds"
534+
// +kubebuilder:validation:Enum:="namespace_egress_bytes_total";"namespace_egress_packets_total";"namespace_ingress_bytes_total";"namespace_ingress_packets_total";"namespace_flows_total";"node_egress_bytes_total";"node_egress_packets_total";"node_ingress_bytes_total";"node_ingress_packets_total";"node_flows_total";"workload_egress_bytes_total";"workload_egress_packets_total";"workload_ingress_bytes_total";"workload_ingress_packets_total";"workload_flows_total";"namespace_drop_bytes_total";"namespace_drop_packets_total";"node_drop_bytes_total";"node_drop_packets_total";"workload_drop_bytes_total";"workload_drop_packets_total";"namespace_rtt_seconds";"node_rtt_seconds";"workload_rtt_seconds";"namespace_dns_latency_seconds";"node_dns_latency_seconds";"workload_dns_latency_seconds";"node_network_policy_events_total";"namespace_network_policy_events_total";"workload_network_policy_events_total"
535535
type FLPMetric string
536536

537537
// `FLPMetrics` define the desired FLP configuration regarding metrics
@@ -547,7 +547,8 @@ type FLPMetrics struct {
547547
// Metrics enabled by default are:
548548
// `namespace_flows_total`, `node_ingress_bytes_total`, `node_egress_bytes_total`, `workload_ingress_bytes_total`,
549549
// `workload_egress_bytes_total`, `namespace_drop_packets_total` (when `PacketDrop` feature is enabled),
550-
// `namespace_rtt_seconds` (when `FlowRTT` feature is enabled), `namespace_dns_latency_seconds` (when `DNSTracking` feature is enabled).
550+
// `namespace_rtt_seconds` (when `FlowRTT` feature is enabled), `namespace_dns_latency_seconds` (when `DNSTracking` feature is enabled),
551+
// `namespace_network_policy_events_total` (when `NetworkEvents` feature is enabled).
551552
// More information, with full list of available metrics: https://github.com/netobserv/network-observability-operator/blob/main/docs/Metrics.md
552553
// +optional
553554
IncludeList *[]FLPMetric `json:"includeList,omitempty"`

‎apis/flowmetrics/v1alpha1/flowmetric_types.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,11 @@ type FlowMetricSpec struct {
9393
// +optional
9494
Labels []string `json:"labels"`
9595

96+
// `flatten` is a list of list-type fields that must be flattened, such as Interfaces and NetworkEvents. Flattened fields generate one metric per item in that field.
97+
// For instance, when flattening `Interfaces` on a bytes counter, a flow having Interfaces [br-ex, ens5] increases one counter for `br-ex` and another for `ens5`.
98+
// +optional
99+
Flatten []string `json:"flatten"`
100+
96101
// Set the `remap` property to use different names for the generated metric labels than the flow fields. Use the origin flow fields as keys, and the desired label names as values.
97102
// +optional
98103
Remap map[string]string `json:"remap"`

‎apis/flowmetrics/v1alpha1/flowmetric_webhook.go‎

Lines changed: 19 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -80,17 +80,17 @@ func checkFlowMetricCartinality(fMetric *FlowMetric) admission.Warnings {
8080
}
8181

8282
func validateFlowMetric(_ context.Context, fMetric *FlowMetric) (admission.Warnings, error) {
83-
var str []string
83+
var fields []string
8484
var allErrs field.ErrorList
8585

8686
for _, f := range fMetric.Spec.Filters {
87-
str = append(str, f.Field)
87+
fields = append(fields, f.Field)
8888
}
8989

90-
if len(str) != 0 {
91-
if !helper.FindFields(str, false) {
92-
allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "filters"), str,
93-
fmt.Sprintf("invalid filter field: %s", str)))
90+
if len(fields) != 0 {
91+
if !helper.FindFields(fields, false) {
92+
allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "filters"), fields,
93+
fmt.Sprintf("invalid filter field: %s", fields)))
9494
}
9595
}
9696

@@ -100,12 +100,13 @@ func validateFlowMetric(_ context.Context, fMetric *FlowMetric) (admission.Warni
100100
fmt.Sprintf("invalid label name: %s", fMetric.Spec.Labels)))
101101
}
102102

103+
labelsMap := make(map[string]any, len(fMetric.Spec.Labels))
104+
for _, label := range fMetric.Spec.Labels {
105+
labelsMap[label] = nil
106+
}
107+
103108
// Only fields defined as Labels are valid for remapping
104109
if len(fMetric.Spec.Remap) != 0 {
105-
labelsMap := make(map[string]any, len(fMetric.Spec.Labels))
106-
for _, label := range fMetric.Spec.Labels {
107-
labelsMap[label] = nil
108-
}
109110
var invalidMapping []string
110111
for toRemap := range fMetric.Spec.Remap {
111112
if _, ok := labelsMap[toRemap]; !ok {
@@ -117,6 +118,14 @@ func validateFlowMetric(_ context.Context, fMetric *FlowMetric) (admission.Warni
117118
fmt.Sprintf("some fields defined for remapping are not defined as labels: %v", invalidMapping)))
118119
}
119120
}
121+
122+
// Check for valid fields
123+
if len(fMetric.Spec.Flatten) != 0 {
124+
if !helper.FindFields(fMetric.Spec.Flatten, false) {
125+
allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "flatten"), fMetric.Spec.Flatten,
126+
fmt.Sprintf("invalid fields to flatten: %s", fMetric.Spec.Flatten)))
127+
}
128+
}
120129
}
121130

122131
if fMetric.Spec.ValueField != "" {

‎apis/flowmetrics/v1alpha1/flowmetric_webhook_test.go‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,27 @@ func TestFlowMetric(t *testing.T) {
105105
},
106106
expectedError: "invalid value field",
107107
},
108+
{
109+
desc: "Valid nested fields",
110+
m: &FlowMetric{
111+
ObjectMeta: metav1.ObjectMeta{
112+
Name: "test1",
113+
Namespace: "test-namespace",
114+
},
115+
Spec: FlowMetricSpec{
116+
Labels: []string{"NetworkEvents>Name"},
117+
Flatten: []string{"NetworkEvents"},
118+
Filters: []MetricFilter{
119+
{
120+
Field: "NetworkEvents>Type",
121+
Value: "acl",
122+
},
123+
},
124+
Remap: map[string]string{"NetworkEvents>Name": "name"},
125+
},
126+
},
127+
expectedError: "",
128+
},
108129
}
109130

110131
for _, test := range tests {

‎apis/flowmetrics/v1alpha1/zz_generated.deepcopy.go‎

Lines changed: 5 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎bundle/manifests/flows.netobserv.io_flowcollectors.yaml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8378,7 +8378,8 @@ spec:
83788378
Metrics enabled by default are:
83798379
`namespace_flows_total`, `node_ingress_bytes_total`, `node_egress_bytes_total`, `workload_ingress_bytes_total`,
83808380
`workload_egress_bytes_total`, `namespace_drop_packets_total` (when `PacketDrop` feature is enabled),
8381-
`namespace_rtt_seconds` (when `FlowRTT` feature is enabled), `namespace_dns_latency_seconds` (when `DNSTracking` feature is enabled).
8381+
`namespace_rtt_seconds` (when `FlowRTT` feature is enabled), `namespace_dns_latency_seconds` (when `DNSTracking` feature is enabled),
8382+
`namespace_network_policy_events_total` (when `NetworkEvents` feature is enabled).
83828383
More information, with full list of available metrics: https://github.com/netobserv/network-observability-operator/blob/main/docs/Metrics.md
83838384
items:
83848385
description: Metric name. More information in https://github.com/netobserv/network-observability-operator/blob/main/docs/Metrics.md.
@@ -8410,6 +8411,9 @@ spec:
84108411
- namespace_dns_latency_seconds
84118412
- node_dns_latency_seconds
84128413
- workload_dns_latency_seconds
8414+
- node_network_policy_events_total
8415+
- namespace_network_policy_events_total
8416+
- workload_network_policy_events_total
84138417
type: string
84148418
type: array
84158419
server:

‎bundle/manifests/flows.netobserv.io_flowmetrics.yaml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -196,6 +196,13 @@ spec:
196196
- matchType
197197
type: object
198198
type: array
199+
flatten:
200+
description: |-
201+
`flatten` is a list of list-type fields that must be flattened, such as Interfaces and NetworkEvents. Flattened fields generate one metric per item in that field.
202+
For instance, when flattening `Interfaces` on a bytes counter, a flow having Interfaces [br-ex, ens5] increases one counter for `br-ex` and another for `ens5`.
203+
items:
204+
type: string
205+
type: array
199206
labels:
200207
description: |-
201208
`labels` is a list of fields that should be used as Prometheus labels, also known as dimensions.

‎config/crd/bases/flows.netobserv.io_flowcollectors.yaml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7733,7 +7733,8 @@ spec:
77337733
Metrics enabled by default are:
77347734
`namespace_flows_total`, `node_ingress_bytes_total`, `node_egress_bytes_total`, `workload_ingress_bytes_total`,
77357735
`workload_egress_bytes_total`, `namespace_drop_packets_total` (when `PacketDrop` feature is enabled),
7736-
`namespace_rtt_seconds` (when `FlowRTT` feature is enabled), `namespace_dns_latency_seconds` (when `DNSTracking` feature is enabled).
7736+
`namespace_rtt_seconds` (when `FlowRTT` feature is enabled), `namespace_dns_latency_seconds` (when `DNSTracking` feature is enabled),
7737+
`namespace_network_policy_events_total` (when `NetworkEvents` feature is enabled).
77377738
More information, with full list of available metrics: https://github.com/netobserv/network-observability-operator/blob/main/docs/Metrics.md
77387739
items:
77397740
description: Metric name. More information in https://github.com/netobserv/network-observability-operator/blob/main/docs/Metrics.md.
@@ -7765,6 +7766,9 @@ spec:
77657766
- namespace_dns_latency_seconds
77667767
- node_dns_latency_seconds
77677768
- workload_dns_latency_seconds
7769+
- node_network_policy_events_total
7770+
- namespace_network_policy_events_total
7771+
- workload_network_policy_events_total
77687772
type: string
77697773
type: array
77707774
server:

‎config/crd/bases/flows.netobserv.io_flowmetrics.yaml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -186,6 +186,13 @@ spec:
186186
- matchType
187187
type: object
188188
type: array
189+
flatten:
190+
description: |-
191+
`flatten` is a list of list-type fields that must be flattened, such as Interfaces and NetworkEvents. Flattened fields generate one metric per item in that field.
192+
For instance, when flattening `Interfaces` on a bytes counter, a flow having Interfaces [br-ex, ens5] increases one counter for `br-ex` and another for `ens5`.
193+
items:
194+
type: string
195+
type: array
189196
labels:
190197
description: |-
191198
`labels` is a list of fields that should be used as Prometheus labels, also known as dimensions.

0 commit comments

Comments
 (0)