Skip to content

Commit 2ea1a7c

Browse files
jotakjotak
authored
NETOBSERV-1875: Make enrichment indexes configurable (#711)
* NETOBSERV-1875: Make enrichment indexes configurable For secondary interfaces, it's better to allow more configurable indexing for pods enrichment. For some network types, pods might be identified by their MAC address advertized in the network-status annotations, while in others it can be by IP, by interface name, or any combination of those. This PR makes it all configurable in that way (example): ```bash secondaryNetworks: - name: my-network index: mac: true ip: true ``` More than one network can be configured that way. Additionally, a new metric is added to track which indexer is used for every enrichment hit: promql: `sum(rate(netobserv_secondary_network_indexer_hit[1m])) by (kind, network, error)` This metric can also report errors and unexpected multiple matches * Add namespace label to metric, add network name enrichment * do not return from building keys when a field is missing: move to next network
1 parent 61bd0fb commit 2ea1a7c

File tree

22 files changed

+529
-328
lines changed

22 files changed

+529
-328
lines changed

‎README.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -422,7 +422,7 @@ parameters:
422422
output: dstLocation
423423
- type: add_kubernetes
424424
kubernetes:
425-
input: srcIP
425+
ipField: srcIP
426426
output: srcK8S
427427
```
428428

@@ -443,7 +443,7 @@ All the geo-location fields will be named by appending `output` value
443443
(e.g., `CountryName`, `CountryLongName`, `RegionName`, `CityName` , `Longitude` and `Latitude`)
444444

445445
The rule `add_kubernetes` generates new fields with kubernetes information by
446-
matching the `input` value (`srcIP` in the example above) with kubernetes `nodes`, `pods` and `services` IPs.
446+
matching the `ipField` value (`srcIP` in the example above) with kubernetes `nodes`, `pods` and `services` IPs.
447447
All the kubernetes fields will be named by appending `output` value
448448
(`srcK8S` in the example above) to the kubernetes metadata field names
449449
(e.g., `Namespace`, `Name`, `Type`, `HostIP`, `OwnerName`, `OwnerType` )

‎cmd/flowlogs-pipeline/main_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ func TestPipelineConfigSetup(t *testing.T) {
5252

5353
js := `{
5454
"PipeLine": "[{\"name\":\"grpc\"},{\"follows\":\"grpc\",\"name\":\"enrich\"},{\"follows\":\"enrich\",\"name\":\"loki\"},{\"follows\":\"enrich\",\"name\":\"prometheus\"}]",
55-
"Parameters": "[{\"ingest\":{\"grpc\":{\"port\":2055},\"type\":\"grpc\"},\"name\":\"grpc\"},{\"name\":\"enrich\",\"transform\":{\"network\":{\"rules\":[{\"kubernetes\":{\"input\":\"SrcAddr\",\"output\":\"SrcK8S\"},\"type\":\"add_kubernetes\"},{\"kubernetes\":{\"input\":\"DstAddr\",\"output\":\"DstK8S\"},\"type\":\"add_kubernetes\"},{\"add_service\":{\"input\":\"DstPort\",\"output\":\"Service\",\"protocol\":\"Proto\"},\"type\":\"add_service\"},{\"add_subnet\":{\"input\":\"SrcAddr\",\"output\":\"SrcSubnet\",\"subnet_mask\":\"/16\"},\"type\":\"add_subnet\"}]},\"type\":\"network\"}},{\"name\":\"loki\",\"write\":{\"loki\":{\"batchSize\":102400,\"batchWait\":\"1s\",\"clientConfig\":{\"follow_redirects\":false,\"proxy_url\":null,\"tls_config\":{\"insecure_skip_verify\":false}},\"labels\":[\"SrcK8S_Namespace\",\"SrcK8S_OwnerName\",\"DstK8S_Namespace\",\"DstK8S_OwnerName\",\"FlowDirection\"],\"maxBackoff\":\"5m0s\",\"maxRetries\":10,\"minBackoff\":\"1s\",\"staticLabels\":{\"app\":\"netobserv-flowcollector\"},\"tenantID\":\"netobserv\",\"timeout\":\"10s\",\"timestampLabel\":\"TimeFlowEndMs\",\"timestampScale\":\"1ms\",\"url\":\"http://loki.netobserv.svc:3100/\"},\"type\":\"loki\"}},{\"encode\":{\"prom\":{\"metrics\":[{\"buckets\":null,\"labels\":[\"Service\",\"SrcK8S_Namespace\"],\"name\":\"bandwidth_per_network_service_per_namespace\",\"type\":\"counter\",\"valueKey\":\"Bytes\"},{\"buckets\":null,\"labels\":[\"SrcSubnet\"],\"name\":\"bandwidth_per_source_subnet\",\"type\":\"counter\",\"valueKey\":\"Bytes\"},{\"buckets\":null,\"labels\":[\"Service\"],\"name\":\"network_service_total\",\"type\":\"counter\",\"valueKey\":\"\"}],\"prefix\":\"netobserv_\"},\"type\":\"prom\"},\"name\":\"prometheus\"}]",
55+
"Parameters": "[{\"ingest\":{\"grpc\":{\"port\":2055},\"type\":\"grpc\"},\"name\":\"grpc\"},{\"name\":\"enrich\",\"transform\":{\"network\":{\"rules\":[{\"kubernetes\":{\"ipField\":\"SrcAddr\",\"output\":\"SrcK8S\"},\"type\":\"add_kubernetes\"},{\"kubernetes\":{\"ipField\":\"DstAddr\",\"output\":\"DstK8S\"},\"type\":\"add_kubernetes\"},{\"add_service\":{\"input\":\"DstPort\",\"output\":\"Service\",\"protocol\":\"Proto\"},\"type\":\"add_service\"},{\"add_subnet\":{\"input\":\"SrcAddr\",\"output\":\"SrcSubnet\",\"subnet_mask\":\"/16\"},\"type\":\"add_subnet\"}]},\"type\":\"network\"}},{\"name\":\"loki\",\"write\":{\"loki\":{\"batchSize\":102400,\"batchWait\":\"1s\",\"clientConfig\":{\"follow_redirects\":false,\"proxy_url\":null,\"tls_config\":{\"insecure_skip_verify\":false}},\"labels\":[\"SrcK8S_Namespace\",\"SrcK8S_OwnerName\",\"DstK8S_Namespace\",\"DstK8S_OwnerName\",\"FlowDirection\"],\"maxBackoff\":\"5m0s\",\"maxRetries\":10,\"minBackoff\":\"1s\",\"staticLabels\":{\"app\":\"netobserv-flowcollector\"},\"tenantID\":\"netobserv\",\"timeout\":\"10s\",\"timestampLabel\":\"TimeFlowEndMs\",\"timestampScale\":\"1ms\",\"url\":\"http://loki.netobserv.svc:3100/\"},\"type\":\"loki\"}},{\"encode\":{\"prom\":{\"metrics\":[{\"buckets\":null,\"labels\":[\"Service\",\"SrcK8S_Namespace\"],\"name\":\"bandwidth_per_network_service_per_namespace\",\"type\":\"counter\",\"valueKey\":\"Bytes\"},{\"buckets\":null,\"labels\":[\"SrcSubnet\"],\"name\":\"bandwidth_per_source_subnet\",\"type\":\"counter\",\"valueKey\":\"Bytes\"},{\"buckets\":null,\"labels\":[\"Service\"],\"name\":\"network_service_total\",\"type\":\"counter\",\"valueKey\":\"\"}],\"prefix\":\"netobserv_\"},\"type\":\"prom\"},\"name\":\"prometheus\"}]",
5656
"Health": {
5757
"Port": "8080"
5858
},

‎contrib/kubernetes/flowlogs-pipeline.conf.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -144,7 +144,7 @@ parameters:
144144
subnet_mask: /16
145145
- type: add_kubernetes
146146
kubernetes:
147-
input: srcIP
147+
ipField: srcIP
148148
output: srcK8S
149149
labels_prefix: srcK8S_labels
150150
- type: add_location

‎docs/api.md‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -250,8 +250,9 @@ Following is the supported API format for network transformations:
250250
name: name of the object
251251
namespace: namespace of the object
252252
kubernetes: Kubernetes rule configuration
253-
input: entry IP input field
254-
mac-input: Optional entry MAC input field
253+
ipField: entry IP input field
254+
interfacesField: entry Interfaces input field
255+
macField: entry MAC input field
255256
output: entry output field
256257
assignee: value needs to assign to output field
257258
labels_prefix: labels prefix to use to copy input lables, if empty labels will not be copied
@@ -272,7 +273,10 @@ Following is the supported API format for network transformations:
272273
protocol: entry protocol field
273274
kubeConfig: global configuration related to Kubernetes (optional)
274275
configPath: path to kubeconfig file (optional)
275-
managedCNI: a list of CNI (network plugins) to manage, for detecting additional interfaces. Currently supported: ovn, multus
276+
secondaryNetworks: configuration for secondary networks
277+
name: name of the secondary network, as mentioned in the annotation 'k8s.v1.cni.cncf.io/network-status'
278+
index: fields to use for indexing, must be any combination of 'mac', 'ip', 'interface'
279+
managedCNI: a list of CNI (network plugins) to manage, for detecting additional interfaces. Currently supported: ovn
276280
servicesFile: path to services file (optional, default: /etc/services)
277281
protocolsFile: path to protocols file (optional, default: /etc/protocols)
278282
subnetLabels: configure subnet and IPs custom labels

‎docs/operational-metrics.md‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,14 @@ Each table below provides documentation for an exported flowlogs-pipeline operat
143143
| **Labels** | stage |
144144

145145

146+
### secondary_network_indexer_hit
147+
| **Name** | secondary_network_indexer_hit |
148+
|:---|:---|
149+
| **Description** | Counter of hits per secondary network index for Kubernetes enrichment |
150+
| **Type** | counter |
151+
| **Labels** | kind, network, warning |
152+
153+
146154
### stage_duration_ms
147155
| **Name** | stage_duration_ms |
148156
|:---|:---|

‎pkg/api/transform_network.go‎

Lines changed: 16 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -39,13 +39,13 @@ func (tn *TransformNetwork) GetServiceFiles() (string, string) {
3939
}
4040

4141
const (
42-
OVN = "ovn"
43-
Multus = "multus"
42+
OVN = "ovn"
4443
)
4544

4645
type NetworkTransformKubeConfig struct {
47-
ConfigPath string `yaml:"configPath,omitempty" json:"configPath,omitempty" doc:"path to kubeconfig file (optional)"`
48-
ManagedCNI []string `yaml:"managedCNI,omitempty" json:"managedCNI,omitempty" doc:"a list of CNI (network plugins) to manage, for detecting additional interfaces. Currently supported: ovn, multus"`
46+
ConfigPath string `yaml:"configPath,omitempty" json:"configPath,omitempty" doc:"path to kubeconfig file (optional)"`
47+
SecondaryNetworks []SecondaryNetwork `yaml:"secondaryNetworks,omitempty" json:"secondaryNetworks,omitempty" doc:"configuration for secondary networks"`
48+
ManagedCNI []string `yaml:"managedCNI,omitempty" json:"managedCNI,omitempty" doc:"a list of CNI (network plugins) to manage, for detecting additional interfaces. Currently supported: ovn"`
4949
}
5050

5151
type TransformNetworkOperationEnum string
@@ -84,12 +84,18 @@ type K8sReference struct {
8484
}
8585

8686
type K8sRule struct {
87-
Input string `yaml:"input,omitempty" json:"input,omitempty" doc:"entry IP input field"`
88-
MacInput string `yaml:"mac-input,omitempty" json:"mac-input,omitempty" doc:"Optional entry MAC input field"`
89-
Output string `yaml:"output,omitempty" json:"output,omitempty" doc:"entry output field"`
90-
Assignee string `yaml:"assignee,omitempty" json:"assignee,omitempty" doc:"value needs to assign to output field"`
91-
LabelsPrefix string `yaml:"labels_prefix,omitempty" json:"labels_prefix,omitempty" doc:"labels prefix to use to copy input lables, if empty labels will not be copied"`
92-
AddZone bool `yaml:"add_zone,omitempty" json:"add_zone,omitempty" doc:"if true the rule will add the zone"`
87+
IPField string `yaml:"ipField,omitempty" json:"ipField,omitempty" doc:"entry IP input field"`
88+
InterfacesField string `yaml:"interfacesField,omitempty" json:"interfacesField,omitempty" doc:"entry Interfaces input field"`
89+
MACField string `yaml:"macField,omitempty" json:"macField,omitempty" doc:"entry MAC input field"`
90+
Output string `yaml:"output,omitempty" json:"output,omitempty" doc:"entry output field"`
91+
Assignee string `yaml:"assignee,omitempty" json:"assignee,omitempty" doc:"value needs to assign to output field"`
92+
LabelsPrefix string `yaml:"labels_prefix,omitempty" json:"labels_prefix,omitempty" doc:"labels prefix to use to copy input lables, if empty labels will not be copied"`
93+
AddZone bool `yaml:"add_zone,omitempty" json:"add_zone,omitempty" doc:"if true the rule will add the zone"`
94+
}
95+
96+
type SecondaryNetwork struct {
97+
Name string `yaml:"name,omitempty" json:"name,omitempty" doc:"name of the secondary network, as mentioned in the annotation 'k8s.v1.cni.cncf.io/network-status'"`
98+
Index map[string]any `yaml:"index,omitempty" json:"index,omitempty" doc:"fields to use for indexing, must be any combination of 'mac', 'ip', 'interface'"`
9399
}
94100

95101
type NetworkGenericRule struct {

‎pkg/confgen/dedup_test.go‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -26,15 +26,15 @@ import (
2626

2727
func Test_dedupeNetworkTransformRules(t *testing.T) {
2828
slice := api.NetworkTransformRules{
29-
api.NetworkTransformRule{Type: "add_kubernetes", Kubernetes: &api.K8sRule{Input: "i1", Output: "o1"}},
30-
api.NetworkTransformRule{Type: "add_kubernetes", Kubernetes: &api.K8sRule{Input: "i2", Output: "o2"}},
31-
api.NetworkTransformRule{Type: "add_kubernetes", Kubernetes: &api.K8sRule{Input: "i3", Output: "o3"}},
32-
api.NetworkTransformRule{Type: "add_kubernetes", Kubernetes: &api.K8sRule{Input: "i2", Output: "o2"}},
29+
api.NetworkTransformRule{Type: "add_kubernetes", Kubernetes: &api.K8sRule{IPField: "i1", Output: "o1"}},
30+
api.NetworkTransformRule{Type: "add_kubernetes", Kubernetes: &api.K8sRule{IPField: "i2", Output: "o2"}},
31+
api.NetworkTransformRule{Type: "add_kubernetes", Kubernetes: &api.K8sRule{IPField: "i3", Output: "o3"}},
32+
api.NetworkTransformRule{Type: "add_kubernetes", Kubernetes: &api.K8sRule{IPField: "i2", Output: "o2"}},
3333
}
3434
expected := api.NetworkTransformRules{
35-
api.NetworkTransformRule{Type: "add_kubernetes", Kubernetes: &api.K8sRule{Input: "i1", Output: "o1"}},
36-
api.NetworkTransformRule{Type: "add_kubernetes", Kubernetes: &api.K8sRule{Input: "i2", Output: "o2"}},
37-
api.NetworkTransformRule{Type: "add_kubernetes", Kubernetes: &api.K8sRule{Input: "i3", Output: "o3"}},
35+
api.NetworkTransformRule{Type: "add_kubernetes", Kubernetes: &api.K8sRule{IPField: "i1", Output: "o1"}},
36+
api.NetworkTransformRule{Type: "add_kubernetes", Kubernetes: &api.K8sRule{IPField: "i2", Output: "o2"}},
37+
api.NetworkTransformRule{Type: "add_kubernetes", Kubernetes: &api.K8sRule{IPField: "i3", Output: "o3"}},
3838
}
3939
actual := dedupeNetworkTransformRules(slice)
4040

‎pkg/config/pipeline_builder_test.go‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -31,14 +31,14 @@ func TestLokiPipeline(t *testing.T) {
3131
pl = pl.TransformNetwork("enrich", api.TransformNetwork{Rules: api.NetworkTransformRules{{
3232
Type: api.NetworkAddKubernetes,
3333
Kubernetes: &api.K8sRule{
34-
Input: "SrcAddr",
35-
Output: "SrcK8S",
34+
IPField: "SrcAddr",
35+
Output: "SrcK8S",
3636
},
3737
}, {
3838
Type: api.NetworkAddKubernetes,
3939
Kubernetes: &api.K8sRule{
40-
Input: "DstAddr",
41-
Output: "DstK8S",
40+
IPField: "DstAddr",
41+
Output: "DstK8S",
4242
},
4343
}}})
4444
pl = pl.WriteLoki("loki", api.WriteLoki{URL: "http://loki:3100/"})
@@ -58,7 +58,7 @@ func TestLokiPipeline(t *testing.T) {
5858

5959
b, err = json.Marshal(params[1])
6060
require.NoError(t, err)
61-
require.JSONEq(t, `{"name":"enrich","transform":{"type":"network","network":{"directionInfo":{},"kubeConfig":{},"rules":[{"kubernetes":{"input":"SrcAddr","output":"SrcK8S"},"type":"add_kubernetes"},{"kubernetes":{"input":"DstAddr","output":"DstK8S"},"type":"add_kubernetes"}]}}}`, string(b))
61+
require.JSONEq(t, `{"name":"enrich","transform":{"type":"network","network":{"directionInfo":{},"kubeConfig":{},"rules":[{"kubernetes":{"ipField":"SrcAddr","output":"SrcK8S"},"type":"add_kubernetes"},{"kubernetes":{"ipField":"DstAddr","output":"DstK8S"},"type":"add_kubernetes"}]}}}`, string(b))
6262

6363
b, err = json.Marshal(params[2])
6464
require.NoError(t, err)
@@ -206,14 +206,14 @@ func TestIPFIXPipeline(t *testing.T) {
206206
pl = pl.TransformNetwork("enrich", api.TransformNetwork{Rules: api.NetworkTransformRules{{
207207
Type: api.NetworkAddKubernetes,
208208
Kubernetes: &api.K8sRule{
209-
Input: "SrcAddr",
210-
Output: "SrcK8S",
209+
IPField: "SrcAddr",
210+
Output: "SrcK8S",
211211
},
212212
}, {
213213
Type: api.NetworkAddKubernetes,
214214
Kubernetes: &api.K8sRule{
215-
Input: "DstAddr",
216-
Output: "DstK8S",
215+
IPField: "DstAddr",
216+
Output: "DstK8S",
217217
},
218218
}}})
219219
pl = pl.WriteIpfix("ipfix", api.WriteIpfix{
@@ -238,7 +238,7 @@ func TestIPFIXPipeline(t *testing.T) {
238238

239239
b, err = json.Marshal(params[1])
240240
require.NoError(t, err)
241-
require.JSONEq(t, `{"name":"enrich","transform":{"type":"network","network":{"directionInfo":{},"kubeConfig":{},"rules":[{"kubernetes":{"input":"SrcAddr","output":"SrcK8S"},"type":"add_kubernetes"},{"kubernetes":{"input":"DstAddr","output":"DstK8S"},"type":"add_kubernetes"}]}}}`, string(b))
241+
require.JSONEq(t, `{"name":"enrich","transform":{"type":"network","network":{"directionInfo":{},"kubeConfig":{},"rules":[{"kubernetes":{"ipField":"SrcAddr","output":"SrcK8S"},"type":"add_kubernetes"},{"kubernetes":{"ipField":"DstAddr","output":"DstK8S"},"type":"add_kubernetes"}]}}}`, string(b))
242242

243243
b, err = json.Marshal(params[2])
244244
require.NoError(t, err)

‎pkg/operational/metrics.go‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,15 @@ var (
8080
TypeHistogram,
8181
"stage",
8282
)
83+
indexerHit = DefineMetric(
84+
"secondary_network_indexer_hit",
85+
"Counter of hits per secondary network index for Kubernetes enrichment",
86+
TypeCounter,
87+
"kind",
88+
"namespace",
89+
"network",
90+
"warning",
91+
)
8392
)
8493

8594
func (def *MetricDefinition) mapLabels(labels []string) prometheus.Labels {
@@ -241,6 +250,10 @@ func (o *Metrics) GetOrCreateStageDurationHisto() *prometheus.HistogramVec {
241250
return o.stageDurationHisto
242251
}
243252

253+
func (o *Metrics) CreateIndexerHitCounter() *prometheus.CounterVec {
254+
return o.NewCounterVec(&indexerHit)
255+
}
256+
244257
func GetDocumentation() string {
245258
doc := ""
246259
sort.Slice(allMetrics, func(i, j int) bool {

‎pkg/pipeline/pipeline_builder.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -434,7 +434,7 @@ func getWriter(opMetrics *operational.Metrics, params config.StageParam) (write.
434434
return writer, err
435435
}
436436

437-
func getTransformer(_ *operational.Metrics, params config.StageParam) (transform.Transformer, error) {
437+
func getTransformer(opMetrics *operational.Metrics, params config.StageParam) (transform.Transformer, error) {
438438
var transformer transform.Transformer
439439
var err error
440440
switch params.Transform.Type {
@@ -443,7 +443,7 @@ func getTransformer(_ *operational.Metrics, params config.StageParam) (transform
443443
case api.FilterType:
444444
transformer, err = transform.NewTransformFilter(params)
445445
case api.NetworkType:
446-
transformer, err = transform.NewTransformNetwork(params)
446+
transformer, err = transform.NewTransformNetwork(params, opMetrics)
447447
case api.NoneType:
448448
transformer, err = transform.NewTransformNone()
449449
default:

0 commit comments

Comments
 (0)