Skip to content

Commit c12294d

Browse files
jpinsonneaujotak
jpinsonneau
and
jotak
authored
NETOBSERV-1927 scopes config (#634)
* scopes config * Update pkg/config/config.go Co-authored-by: Joel Takvorian <[email protected]> * fix compile * scope & groups types * mandatory fields * missing fields in go type * fixes * step into * fix step into + add tests --------- Co-authored-by: Joel Takvorian <[email protected]>
1 parent dfe4f95 commit c12294d

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

45 files changed

+819
-757
lines changed

‎config/sample-config.yaml‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -980,6 +980,126 @@ frontend:
980980
name: Flow RTT
981981
component: number
982982
hint: Specify a TCP smoothed Round Trip Time in nanoseconds.
983+
scopes:
984+
- id: cluster
985+
name: Cluster
986+
shortName: Cl
987+
description: Cluster name or identifier
988+
labels:
989+
- K8S_ClusterName
990+
feature: multiCluster
991+
filter: cluster_name
992+
stepInto: zone
993+
- id: zone
994+
name: Zone
995+
shortName: AZ
996+
description: Availability zone
997+
labels:
998+
- SrcK8S_Zone
999+
- DstK8S_Zone
1000+
feature: zones
1001+
groups:
1002+
- clusters
1003+
filters:
1004+
- src_zone
1005+
- dst_zone
1006+
stepInto: host
1007+
- id: host
1008+
name: Node
1009+
description: Node on which the resources are running
1010+
labels:
1011+
- SrcK8S_HostName
1012+
- DstK8S_HostName
1013+
groups:
1014+
- clusters
1015+
- zones
1016+
- clusters+zones
1017+
filters:
1018+
- src_host_name
1019+
- dst_host_name
1020+
stepInto: resource
1021+
- id: namespace
1022+
name: Namespace
1023+
shortName: NS
1024+
description: Resource namespace
1025+
labels:
1026+
- SrcK8S_Namespace
1027+
- DstK8S_Namespace
1028+
groups:
1029+
- clusters
1030+
- clusters+zones
1031+
- clusters+hosts
1032+
- zones
1033+
- zones+hosts
1034+
- hosts
1035+
filters:
1036+
- src_namespace
1037+
- dst_namespace
1038+
stepInto: owner
1039+
- id: owner
1040+
name: Owner
1041+
shortName: Own
1042+
description: Controller owner, such as a Deployment
1043+
labels:
1044+
- SrcK8S_OwnerName
1045+
- SrcK8S_OwnerType
1046+
- DstK8S_OwnerName
1047+
- DstK8S_OwnerType
1048+
- SrcK8S_Namespace
1049+
- DstK8S_Namespace
1050+
groups:
1051+
- clusters
1052+
- clusters+zones
1053+
- clusters+hosts
1054+
- clusters+namespaces
1055+
- zones
1056+
- zones+hosts
1057+
- zones+namespaces
1058+
- hosts
1059+
- hosts+namespaces
1060+
- namespaces
1061+
filters:
1062+
- src_owner_name
1063+
- dst_owner_name
1064+
stepInto: resource
1065+
- id: resource
1066+
name: Resource
1067+
shortName: Res
1068+
description: Base resource, such as a Pod, a Service or a Node
1069+
labels:
1070+
- SrcK8S_Name
1071+
- SrcK8S_Type
1072+
- SrcK8S_OwnerName
1073+
- SrcK8S_OwnerType
1074+
- SrcK8S_Namespace
1075+
- SrcAddr
1076+
- SrcK8S_HostName
1077+
- DstK8S_Name
1078+
- DstK8S_Type
1079+
- DstK8S_OwnerName
1080+
- DstK8S_OwnerType
1081+
- DstK8S_Namespace
1082+
- DstAddr
1083+
- DstK8S_HostName
1084+
groups:
1085+
- clusters
1086+
- clusters+zones
1087+
- clusters+hosts
1088+
- clusters+namespaces
1089+
- clusters+owners
1090+
- zones
1091+
- zones+hosts
1092+
- zones+namespaces
1093+
- zones+owners
1094+
- hosts
1095+
- hosts+namespaces
1096+
- hosts+owners
1097+
- namespaces
1098+
- namespaces+owners
1099+
- owners
1100+
filters:
1101+
- src_resource
1102+
- dst_resource
9831103
fields:
9841104
- name: TimeFlowStartMs
9851105
type: number

‎pkg/config/config.go‎

Lines changed: 35 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ type Column struct {
7777
Filter string `yaml:"filter,omitempty" json:"filter,omitempty"`
7878
Default bool `yaml:"default,omitempty" json:"default,omitempty"`
7979
Width int `yaml:"width,omitempty" json:"width,omitempty"`
80-
Feature string `yaml:"feature" json:"feature"`
80+
Feature string `yaml:"feature,omitempty" json:"feature,omitempty"`
8181
}
8282

8383
type Filter struct {
@@ -92,6 +92,19 @@ type Filter struct {
9292
Placeholder string `yaml:"placeholder,omitempty" json:"placeholder,omitempty"`
9393
}
9494

95+
type Scope struct {
96+
ID string `yaml:"id" json:"id"`
97+
Name string `yaml:"name" json:"name"`
98+
ShortName string `yaml:"shortName" json:"shortName"`
99+
Description string `yaml:"description" json:"description"`
100+
Labels []string `yaml:"labels" json:"labels"`
101+
Feature string `yaml:"feature,omitempty" json:"feature,omitempty"`
102+
Groups []string `yaml:"groups,omitempty" json:"groups,omitempty"`
103+
Filter string `yaml:"filter,omitempty" json:"filter,omitempty"`
104+
Filters []string `yaml:"filters,omitempty" json:"filters,omitempty"`
105+
StepInto string `yaml:"stepInto,omitempty" json:"stepInto,omitempty"`
106+
}
107+
95108
type QuickFilter struct {
96109
Name string `yaml:"name" json:"name"`
97110
Filter map[string]string `yaml:"filter" json:"filter"`
@@ -120,6 +133,7 @@ type Frontend struct {
120133
Panels []string `yaml:"panels" json:"panels"`
121134
Columns []Column `yaml:"columns" json:"columns"`
122135
Filters []Filter `yaml:"filters" json:"filters"`
136+
Scopes []Scope `yaml:"scopes" json:"scopes"`
123137
QuickFilters []QuickFilter `yaml:"quickFilters" json:"quickFilters"`
124138
AlertNamespaces []string `yaml:"alertNamespaces" json:"alertNamespaces"`
125139
Sampling int `yaml:"sampling" json:"sampling"`
@@ -166,7 +180,13 @@ func ReadFile(version, date, filename string) (*Config, error) {
166180
{ID: "SrcAddr", Name: "IP", Group: "Source", Field: "SrcAddr", Default: true, Width: 15},
167181
{ID: "DstAddr", Name: "IP", Group: "Destination", Field: "DstAddr", Default: true, Width: 15},
168182
},
169-
Filters: []Filter{},
183+
Filters: []Filter{},
184+
Scopes: []Scope{
185+
{ID: "host", Name: "Node", Labels: []string{"SrcK8S_HostName", "DstK8S_HostName"}},
186+
{ID: "namespace", Name: "Namespace", Labels: []string{"SrcK8S_Namespace", "DstK8S_Namespace"}},
187+
{ID: "owner", Name: "Owner", Labels: []string{"SrcK8S_OwnerName", "SrcK8S_OwnerType", "DstK8S_OwnerName", "DstK8S_OwnerType", "SrcK8S_Namespace", "DstK8S_Namespace"}},
188+
{ID: "resource", Name: "Resource", Labels: []string{"SrcK8S_Name", "SrcK8S_Type", "SrcK8S_OwnerName", "SrcK8S_OwnerType", "SrcK8S_Namespace", "SrcAddr", "SrcK8S_HostName", "DstK8S_Name", "DstK8S_Type", "DstK8S_OwnerName", "DstK8S_OwnerType", "DstK8S_Namespace", "DstAddr", "DstK8S_HostName"}},
189+
},
170190
QuickFilters: []QuickFilter{},
171191
Features: []string{},
172192
Deduper: Deduper{
@@ -299,3 +319,16 @@ func (c *Config) GetAuthChecker() (auth.Checker, error) {
299319
}
300320
return auth.NewChecker(checkType, client.NewInCluster)
301321
}
322+
323+
func (c *Frontend) GetAggregateKeyLabels() map[string][]string {
324+
keyLabels := map[string][]string{
325+
"app": {"app"},
326+
"droppedState": {"PktDropLatestState"},
327+
"droppedCause": {"PktDropLatestDropCause"},
328+
"dnsRCode": {"DnsFlagsResponseCode"},
329+
}
330+
for i := range c.Scopes {
331+
keyLabels[c.Scopes[i].ID] = c.Scopes[i].Labels
332+
}
333+
return keyLabels
334+
}

‎pkg/handler/lokiclientmock/loki_client_mock.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ func (o *LokiClientMock) Get(url string) ([]byte, int, error) {
3737
} else if strings.Contains(url, "by(K8S_ClusterName)") {
3838
path += "_cluster.json"
3939
} else if strings.Contains(url, "by(SrcK8S_Zone,DstK8S_Zone)") {
40-
path += "zone.json"
40+
path += "_zone.json"
4141
} else if strings.Contains(url, "by(SrcK8S_HostName,DstK8S_HostName)") {
4242
path += "_host.json"
4343
} else if strings.Contains(url, "by(SrcK8S_Namespace,DstK8S_Namespace)") {

‎pkg/handler/topology.go‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -139,7 +139,7 @@ func (h *Handlers) extractTopologyQueryParams(params url.Values, ds constants.Da
139139
namespace,
140140
func(filters filters.SingleQuery) bool {
141141
// Do not expand if this is managed from prometheus
142-
sr, _ := getEligiblePromMetric(h.PromInventory, filters, &in, namespace != "")
142+
sr, _ := getEligiblePromMetric(h.Cfg.Frontend.GetAggregateKeyLabels(), h.PromInventory, filters, &in, namespace != "")
143143
return sr != nil && len(sr.Found) > 0
144144
},
145145
)
@@ -276,12 +276,12 @@ func buildTopologyQuery(
276276
qr *v1.Range,
277277
isDev bool,
278278
) (string, *prometheus.Query, int, error) {
279-
search, unsupportedReason := getEligiblePromMetric(promInventory, filters, in, isDev)
279+
search, unsupportedReason := getEligiblePromMetric(cfg.Frontend.GetAggregateKeyLabels(), promInventory, filters, in, isDev)
280280
if unsupportedReason != "" {
281281
hlog.Debugf("Unsupported Prometheus query; reason: %s.", unsupportedReason)
282282
} else if search != nil && len(search.Found) > 0 {
283283
// Success, we can use Prometheus
284-
qb := prometheus.NewQuery(in, qr, filters, search.Found)
284+
qb := prometheus.NewQuery(cfg.Frontend.GetAggregateKeyLabels(), in, qr, filters, search.Found)
285285
q := qb.Build()
286286
return "", &q, http.StatusOK, nil
287287
}
@@ -310,7 +310,7 @@ func buildTopologyQuery(
310310
"this request could not be performed with Prometheus metrics%s: it requires installing and enabling Loki", reason)
311311
}
312312

313-
qb, err := loki.NewTopologyQuery(&cfg.Loki, in)
313+
qb, err := loki.NewTopologyQuery(&cfg.Loki, cfg.Frontend.GetAggregateKeyLabels(), in)
314314
if err != nil {
315315
return "", nil, http.StatusBadRequest, err
316316
}
@@ -321,7 +321,7 @@ func buildTopologyQuery(
321321
return EncodeQuery(qb.Build()), nil, http.StatusOK, nil
322322
}
323323

324-
func getEligiblePromMetric(promInventory *prometheus.Inventory, filters filters.SingleQuery, in *loki.TopologyInput, isDev bool) (*prometheus.SearchResult, string) {
324+
func getEligiblePromMetric(kl map[string][]string, promInventory *prometheus.Inventory, filters filters.SingleQuery, in *loki.TopologyInput, isDev bool) (*prometheus.SearchResult, string) {
325325
if in.DataSource != constants.DataSourceAuto && in.DataSource != constants.DataSourceProm {
326326
return nil, ""
327327
}
@@ -332,7 +332,7 @@ func getEligiblePromMetric(promInventory *prometheus.Inventory, filters filters.
332332
return nil, fmt.Sprintf("RecordType not managed: %s", in.RecordType)
333333
}
334334

335-
labelsNeeded, _ := prometheus.GetLabelsAndFilter(in.Aggregate, in.Groups)
335+
labelsNeeded, _ := prometheus.GetLabelsAndFilter(kl, in.Aggregate, in.Groups)
336336
fromFilters, unsupportedReason := prometheus.FiltersToLabels(filters)
337337
if unsupportedReason != "" {
338338
return nil, unsupportedReason

‎pkg/loki/topology_query.go‎

Lines changed: 10 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -13,28 +13,6 @@ const (
1313
topologyDefaultLimit = "100"
1414
)
1515

16-
var (
17-
aggregateKeyLabels = map[string][]string{
18-
"app": {"app"},
19-
"droppedState": {"PktDropLatestState"},
20-
"droppedCause": {"PktDropLatestDropCause"},
21-
"dnsRCode": {"DnsFlagsResponseCode"},
22-
"cluster": {"K8S_ClusterName"},
23-
"zone": {"SrcK8S_Zone", "DstK8S_Zone"},
24-
"host": {"SrcK8S_HostName", "DstK8S_HostName"},
25-
"namespace": {"SrcK8S_Namespace", "DstK8S_Namespace"},
26-
"owner": {"SrcK8S_OwnerName", "SrcK8S_OwnerType", "DstK8S_OwnerName", "DstK8S_OwnerType", "SrcK8S_Namespace", "DstK8S_Namespace"},
27-
"resource": {"SrcK8S_Name", "SrcK8S_Type", "SrcK8S_OwnerName", "SrcK8S_OwnerType", "SrcK8S_Namespace", "SrcAddr", "SrcK8S_HostName", "DstK8S_Name", "DstK8S_Type", "DstK8S_OwnerName", "DstK8S_OwnerType", "DstK8S_Namespace", "DstAddr", "DstK8S_HostName"},
28-
}
29-
groupKeyLabels = map[string][]string{
30-
"clusters": {"K8S_ClusterName"},
31-
"zones": {"SrcK8S_Zone", "DstK8S_Zone"},
32-
"hosts": {"SrcK8S_HostName", "DstK8S_HostName"},
33-
"namespaces": {"SrcK8S_Namespace", "DstK8S_Namespace"},
34-
"owners": {"SrcK8S_OwnerName", "SrcK8S_OwnerType", "DstK8S_OwnerName", "DstK8S_OwnerType"},
35-
}
36-
)
37-
3816
type TopologyInput struct {
3917
Start string
4018
End string
@@ -53,10 +31,11 @@ type TopologyInput struct {
5331

5432
type TopologyQueryBuilder struct {
5533
*FlowQueryBuilder
56-
topology *TopologyInput
34+
topology *TopologyInput
35+
aggregateKeyLabels map[string][]string
5736
}
5837

59-
func NewTopologyQuery(cfg *config.Loki, in *TopologyInput) (*TopologyQueryBuilder, error) {
38+
func NewTopologyQuery(cfg *config.Loki, kl map[string][]string, in *TopologyInput) (*TopologyQueryBuilder, error) {
6039
var dedup bool
6140
var rt constants.RecordType
6241
if slices.Contains(constants.AnyConnectionType, string(in.RecordType)) {
@@ -69,20 +48,21 @@ func NewTopologyQuery(cfg *config.Loki, in *TopologyInput) (*TopologyQueryBuilde
6948

7049
fqb := NewFlowQueryBuilder(cfg, in.Start, in.End, in.Top, dedup, rt, in.PacketLoss)
7150
return &TopologyQueryBuilder{
72-
FlowQueryBuilder: fqb,
73-
topology: in,
51+
FlowQueryBuilder: fqb,
52+
topology: in,
53+
aggregateKeyLabels: kl,
7454
}, nil
7555
}
7656

77-
func GetLabelsAndFilter(aggregate, groups string) ([]string, string) {
57+
func GetLabelsAndFilter(kl map[string][]string, aggregate, groups string) ([]string, string) {
7858
var fields []string
7959
var filter string
80-
if fields = aggregateKeyLabels[aggregate]; fields == nil {
60+
if fields = kl[aggregate]; fields == nil {
8161
fields = []string{aggregate}
8262
filter = aggregate
8363
}
8464
if groups != "" {
85-
for gr, labels := range groupKeyLabels {
65+
for gr, labels := range kl {
8666
if strings.Contains(groups, gr) {
8767
for _, label := range labels {
8868
if !slices.Contains(fields, label) {
@@ -142,7 +122,7 @@ func (q *TopologyQueryBuilder) Build() string {
142122
top = topologyDefaultLimit
143123
}
144124

145-
labels, extraFilter := GetLabelsAndFilter(q.topology.Aggregate, q.topology.Groups)
125+
labels, extraFilter := GetLabelsAndFilter(q.aggregateKeyLabels, q.topology.Aggregate, q.topology.Groups)
146126
if q.config.IsLabel(extraFilter) {
147127
extraFilter = ""
148128
}

‎pkg/loki/topology_query_test.go‎

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,19 @@ var lokiConfig = config.Loki{
1414
Labels: []string{"SrcK8S_Namespace", "SrcK8S_OwnerName", "DstK8S_Namespace", "DstK8S_OwnerName", "FlowDirection"},
1515
}
1616

17+
var aggregateKeyLabels = map[string][]string{
18+
"app": {"app"},
19+
"droppedState": {"PktDropLatestState"},
20+
"droppedCause": {"PktDropLatestDropCause"},
21+
"dnsRCode": {"DnsFlagsResponseCode"},
22+
"cluster": {"K8S_ClusterName"},
23+
"zone": {"SrcK8S_Zone", "DstK8S_Zone"},
24+
"host": {"SrcK8S_HostName", "DstK8S_HostName"},
25+
"namespace": {"SrcK8S_Namespace", "DstK8S_Namespace"},
26+
"owner": {"SrcK8S_OwnerName", "SrcK8S_OwnerType", "DstK8S_OwnerName", "DstK8S_OwnerType", "SrcK8S_Namespace", "DstK8S_Namespace"},
27+
"resource": {"SrcK8S_Name", "SrcK8S_Type", "SrcK8S_OwnerName", "SrcK8S_OwnerType", "SrcK8S_Namespace", "SrcAddr", "SrcK8S_HostName", "DstK8S_Name", "DstK8S_Type", "DstK8S_OwnerName", "DstK8S_OwnerType", "DstK8S_Namespace", "DstAddr", "DstK8S_HostName"},
28+
}
29+
1730
func TestBuildTopologyQuery_SimpleAggregate(t *testing.T) {
1831
in := TopologyInput{
1932
Start: "(start)",
@@ -28,7 +41,7 @@ func TestBuildTopologyQuery_SimpleAggregate(t *testing.T) {
2841
Aggregate: "namespace",
2942
DedupMark: true,
3043
}
31-
q, err := NewTopologyQuery(&lokiConfig, &in)
44+
q, err := NewTopologyQuery(&lokiConfig, aggregateKeyLabels, &in)
3245
require.NoError(t, err)
3346
result := q.Build()
3447
assert.Equal(
@@ -54,7 +67,7 @@ func TestBuildTopologyQuery_GroupsAndAggregate(t *testing.T) {
5467
Groups: "hosts",
5568
DedupMark: true,
5669
}
57-
q, err := NewTopologyQuery(&lokiConfig, &in)
70+
q, err := NewTopologyQuery(&lokiConfig, aggregateKeyLabels, &in)
5871
require.NoError(t, err)
5972
result := q.Build()
6073
assert.Equal(
@@ -79,7 +92,7 @@ func TestBuildTopologyQuery_CustomAggregate(t *testing.T) {
7992
Aggregate: "SomeField",
8093
DedupMark: true,
8194
}
82-
q, err := NewTopologyQuery(&lokiConfig, &in)
95+
q, err := NewTopologyQuery(&lokiConfig, aggregateKeyLabels, &in)
8396
require.NoError(t, err)
8497
result := q.Build()
8598
assert.Equal(
@@ -104,7 +117,7 @@ func TestBuildTopologyQuery_CustomLabelAggregate(t *testing.T) {
104117
Aggregate: "FlowDirection",
105118
DedupMark: true,
106119
}
107-
q, err := NewTopologyQuery(&lokiConfig, &in)
120+
q, err := NewTopologyQuery(&lokiConfig, aggregateKeyLabels, &in)
108121
require.NoError(t, err)
109122
result := q.Build()
110123
assert.Equal(

0 commit comments

Comments
 (0)