Skip to content

Commit 07ebba8

Browse files
author
Mario Macias
authored
NETOBSERV-223: filter by empty names (#166)
* 2-step line filter composition * added tests * fixed or conditions in regexpes * add extra test to isolate bug on frontend * added extra code comments
1 parent feb6b88 commit 07ebba8

File tree

3 files changed

+149
-54
lines changed

3 files changed

+149
-54
lines changed

‎pkg/loki/filter.go‎

Lines changed: 73 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,9 @@ import (
55
"strings"
66
)
77

8+
// remove quotes and replace * by regex any
9+
var valueReplacer = strings.NewReplacer(`*`, `.*`, `"`, "")
10+
811
type labelMatcher string
912

1013
const (
@@ -30,6 +33,17 @@ type labelFilter struct {
3033
valueType valueType
3134
}
3235

36+
// lineFilter represents a condition based on a JSON raw text match.
37+
type lineFilter struct {
38+
key string
39+
values []lineMatch
40+
}
41+
42+
type lineMatch struct {
43+
value string
44+
valueType valueType
45+
}
46+
3347
func stringLabelFilter(labelKey string, value string) labelFilter {
3448
return labelFilter{
3549
key: labelKey,
@@ -72,10 +86,67 @@ func (f *labelFilter) writeInto(sb *strings.Builder) {
7286
sb.WriteString(f.value)
7387
sb.WriteString(`")`)
7488
case typeRegex:
75-
sb.WriteByte('`')
89+
sb.WriteString("`(?i).*")
7690
sb.WriteString(f.value)
77-
sb.WriteByte('`')
91+
sb.WriteString(".*`")
7892
default:
7993
panic(fmt.Sprint("wrong filter value type", int(f.valueType)))
8094
}
8195
}
96+
97+
// asLabelFilters transforms a lineFilter (raw text match) into a group of
98+
// labelFilters (attributes match)
99+
func (f *lineFilter) asLabelFilters() []labelFilter {
100+
lfs := make([]labelFilter, 0, len(f.values))
101+
for _, v := range f.values {
102+
lf := labelFilter{
103+
key: f.key,
104+
valueType: v.valueType,
105+
value: v.value,
106+
}
107+
if v.valueType == typeRegex {
108+
lf.matcher = labelMatches
109+
} else {
110+
lf.matcher = labelEqual
111+
}
112+
lfs = append(lfs, lf)
113+
}
114+
return lfs
115+
}
116+
117+
// writeInto transforms a lineFilter to its corresponding part of a LogQL query
118+
// under construction (contained in the provided strings.Builder)
119+
func (f *lineFilter) writeInto(sb *strings.Builder) {
120+
for i, v := range f.values {
121+
if i > 0 {
122+
sb.WriteByte('|')
123+
}
124+
// match end of KEY + regex VALUE:
125+
// if numeric, KEY":VALUE,
126+
// if string KEY":"VALUE"
127+
// ie 'Port' key will match both 'SrcPort":"XXX"' and 'DstPort":"XXX"
128+
// VALUE can be quoted for exact match or contains * to inject regex any
129+
// For numeric values, exact match is implicit
130+
// (the trick is to match for the ending coma; it works as long as the filtered field
131+
// is not the last one (they're in alphabetic order); a less performant alternative
132+
// but more future-proof/less hacky could be to move that to a json filter, if needed)
133+
sb.WriteString(f.key)
134+
sb.WriteString(`":`)
135+
switch v.valueType {
136+
case typeNumber:
137+
sb.WriteString(v.value)
138+
// a number can be followed by } if it's the last property of a JSON document
139+
sb.WriteString("[,}]")
140+
case typeString, typeIP:
141+
// exact matches are specified as just strings
142+
sb.WriteByte('"')
143+
sb.WriteString(valueReplacer.Replace(v.value))
144+
sb.WriteByte('"')
145+
// contains-match are specified as regular expressions
146+
case typeRegex:
147+
sb.WriteString(`"(?i)[^"]*`)
148+
sb.WriteString(valueReplacer.Replace(v.value))
149+
sb.WriteString(`.*"`)
150+
}
151+
}
152+
}

‎pkg/loki/flow_query.go‎

Lines changed: 27 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -21,17 +21,14 @@ const (
2121
// can contains only alphanumeric / '-' / '_' / '.' / ',' / '"' / '*' / ':' / '/' characteres
2222
var filterRegexpValidation = regexp.MustCompile(`^[\w-_.,\"*:/]*$`)
2323

24-
// remove quotes and replace * by regex any
25-
var valueReplacer = strings.NewReplacer(`*`, `.*`, `"`, "")
26-
2724
// FlowQueryBuilder stores a state to build a LogQL query
2825
type FlowQueryBuilder struct {
2926
config *Config
3027
startTime string
3128
endTime string
3229
limit string
3330
labelFilters []labelFilter
34-
lineFilters []string
31+
lineFilters []lineFilter
3532
jsonFilters [][]labelFilter
3633
}
3734

@@ -123,44 +120,33 @@ func (q *FlowQueryBuilder) addLabelRegex(key string, values []string) {
123120
}
124121

125122
func (q *FlowQueryBuilder) addLineFilters(key string, values []string) {
126-
regexStr := strings.Builder{}
127-
for i, value := range values {
128-
if i > 0 {
129-
regexStr.WriteByte('|')
130-
}
131-
// match end of KEY + regex VALUE:
132-
// if numeric, KEY":VALUE,
133-
// if string KEY":"VALUE"
134-
// ie 'Port' key will match both 'SrcPort":"XXX"' and 'DstPort":"XXX"
135-
// VALUE can be quoted for exact match or contains * to inject regex any
136-
// For numeric values, exact match is implicit
137-
// (the trick is to match for the ending coma; it works as long as the filtered field
138-
// is not the last one (they're in alphabetic order); a less performant alternative
139-
// but more future-proof/less hacky could be to move that to a json filter, if needed)
140-
regexStr.WriteString(key)
141-
regexStr.WriteString(`":`)
142-
if fields.IsNumeric(key) {
143-
regexStr.WriteString(value)
144-
regexStr.WriteByte(',')
145-
} else {
146-
regexStr.WriteByte('"')
147-
// match start any if not quoted
148-
// and case insensitive
149-
if !strings.HasPrefix(value, `"`) {
150-
regexStr.WriteString("(?i)[^\"]*")
151-
}
152-
//inject value with regex
153-
regexStr.WriteString(valueReplacer.Replace(value))
154-
// match end any if not quoted
155-
if !strings.HasSuffix(value, `"`) {
156-
regexStr.WriteString(".*")
157-
}
158-
regexStr.WriteByte('"')
123+
if len(values) == 0 {
124+
return
125+
}
126+
lf := lineFilter{
127+
key: key,
128+
}
129+
isNumeric := fields.IsNumeric(key)
130+
emptyMatches := false
131+
for _, value := range values {
132+
lm := lineMatch{}
133+
switch {
134+
case isNumeric:
135+
lm = lineMatch{valueType: typeNumber, value: value}
136+
case isExactMatch(value):
137+
lm = lineMatch{valueType: typeString, value: trimExactMatch(value)}
138+
emptyMatches = emptyMatches || len(lm.value) == 0
139+
default:
140+
lm = lineMatch{valueType: typeRegex, value: value}
159141
}
142+
lf.values = append(lf.values, lm)
160143
}
161-
162-
if regexStr.Len() > 0 {
163-
q.lineFilters = append(q.lineFilters, regexStr.String())
144+
// if there is at least an empty exact match, there is no uniform/safe way to filter by text,
145+
// so we should use JSON label matchers instead of text line matchers
146+
if emptyMatches {
147+
q.jsonFilters = append(q.jsonFilters, lf.asLabelFilters())
148+
} else {
149+
q.lineFilters = append(q.lineFilters, lf)
164150
}
165151
}
166152

@@ -195,7 +181,7 @@ func (q *FlowQueryBuilder) appendLabels(sb *strings.Builder) {
195181
func (q *FlowQueryBuilder) appendLineFilters(sb *strings.Builder) {
196182
for _, lf := range q.lineFilters {
197183
sb.WriteString("|~`")
198-
sb.WriteString(lf)
184+
lf.writeInto(sb)
199185
sb.WriteByte('`')
200186
}
201187
}

‎pkg/server/server_flows_test.go‎

Lines changed: 49 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -38,19 +38,19 @@ func TestLokiFiltering(t *testing.T) {
3838
inputPath: "?filters=" + url.QueryEscape("Proto=6&SrcK8S_Name=test"),
3939
outputQueryParts: []string{
4040
"?query={app=\"netobserv-flowcollector\"}",
41-
"|~`Proto\":6,`",
41+
"|~`Proto\":6[,}]`",
4242
"|~`SrcK8S_Name\":\"(?i)[^\"]*test.*\"`",
4343
},
4444
}, {
4545
inputPath: "?filters=" + url.QueryEscape("Proto=6|SrcK8S_Name=test"),
4646
outputQueries: []string{
47-
"?query={app=\"netobserv-flowcollector\"}|~`Proto\":6,`",
47+
"?query={app=\"netobserv-flowcollector\"}|~`Proto\":6[,}]`",
4848
"?query={app=\"netobserv-flowcollector\"}|~`SrcK8S_Name\":\"(?i)[^\"]*test.*\"`",
4949
},
5050
}, {
5151
inputPath: "?filters=" + url.QueryEscape("Proto=6|SrcK8S_Name=test") + "&reporter=source",
5252
outputQueries: []string{
53-
"?query={app=\"netobserv-flowcollector\",FlowDirection=\"1\"}|~`Proto\":6,`",
53+
"?query={app=\"netobserv-flowcollector\",FlowDirection=\"1\"}|~`Proto\":6[,}]`",
5454
"?query={app=\"netobserv-flowcollector\",FlowDirection=\"1\"}|~`SrcK8S_Name\":\"(?i)[^\"]*test.*\"`",
5555
},
5656
}, {
@@ -71,7 +71,7 @@ func TestLokiFiltering(t *testing.T) {
7171
}, {
7272
inputPath: "?filters=" + url.QueryEscape("SrcPort=8080&SrcAddr=10.128.0.1&SrcK8S_Namespace=default"),
7373
outputQueries: []string{
74-
"?query={app=\"netobserv-flowcollector\",SrcK8S_Namespace=~\"(?i).*default.*\"}|~`SrcPort\":8080,`|json|SrcAddr=ip(\"10.128.0.1\")",
74+
"?query={app=\"netobserv-flowcollector\",SrcK8S_Namespace=~\"(?i).*default.*\"}|~`SrcPort\":8080[,}]`|json|SrcAddr=ip(\"10.128.0.1\")",
7575
},
7676
}, {
7777
inputPath: "?filters=" + url.QueryEscape("SrcAddr=10.128.0.1&DstAddr=10.128.0.2"),
@@ -90,14 +90,14 @@ func TestLokiFiltering(t *testing.T) {
9090
outputQueries: []string{
9191
"?query={app=\"netobserv-flowcollector\",SrcK8S_Namespace=~\"(?i).*default.*\"}",
9292
"?query={app=\"netobserv-flowcollector\"}|json|SrcAddr=ip(\"10.128.0.1\")",
93-
"?query={app=\"netobserv-flowcollector\"}|~`SrcPort\":8080,`",
93+
"?query={app=\"netobserv-flowcollector\"}|~`SrcPort\":8080[,}]`",
9494
},
9595
}, {
9696
inputPath: "?filters=" + url.QueryEscape("SrcPort=8080|SrcAddr=10.128.0.1|SrcK8S_Namespace=default") + "&reporter=destination",
9797
outputQueries: []string{
9898
"?query={app=\"netobserv-flowcollector\",FlowDirection=\"0\",SrcK8S_Namespace=~\"(?i).*default.*\"}",
9999
"?query={app=\"netobserv-flowcollector\",FlowDirection=\"0\"}|json|SrcAddr=ip(\"10.128.0.1\")",
100-
"?query={app=\"netobserv-flowcollector\",FlowDirection=\"0\"}|~`SrcPort\":8080,`",
100+
"?query={app=\"netobserv-flowcollector\",FlowDirection=\"0\"}|~`SrcPort\":8080[,}]`",
101101
},
102102
}, {
103103
inputPath: "?startTime=1640991600",
@@ -125,27 +125,65 @@ func TestLokiFiltering(t *testing.T) {
125125
inputPath: "?filters=" + url.QueryEscape("Port=8080&K8S_Name=test"),
126126
outputQueryParts: []string{
127127
"?query={app=\"netobserv-flowcollector\"}",
128-
"|~`Port\":8080,`",
128+
"|~`Port\":8080[,}]`",
129129
"|~`K8S_Name\":\"(?i)[^\"]*test.*\"`",
130130
},
131131
}, {
132132
inputPath: "?filters=" + url.QueryEscape("Port=8080|K8S_Name=test"),
133133
outputQueries: []string{
134134
"?query={app=\"netobserv-flowcollector\"}|~`K8S_Name\":\"(?i)[^\"]*test.*\"`",
135-
"?query={app=\"netobserv-flowcollector\"}|~`Port\":8080,`",
135+
"?query={app=\"netobserv-flowcollector\"}|~`Port\":8080[,}]`",
136136
},
137137
}, {
138138
inputPath: "?filters=" + url.QueryEscape("Port=8080&SrcK8S_Namespace=test|Port=8080&DstK8S_Namespace=test"),
139139
outputQueries: []string{
140-
"?query={app=\"netobserv-flowcollector\",SrcK8S_Namespace=~\"(?i).*test.*\"}|~`Port\":8080,`",
141-
"?query={app=\"netobserv-flowcollector\",DstK8S_Namespace=~\"(?i).*test.*\"}|~`Port\":8080,`",
140+
"?query={app=\"netobserv-flowcollector\",SrcK8S_Namespace=~\"(?i).*test.*\"}|~`Port\":8080[,}]`",
141+
"?query={app=\"netobserv-flowcollector\",DstK8S_Namespace=~\"(?i).*test.*\"}|~`Port\":8080[,}]`",
142142
},
143143
}, {
144144
inputPath: "?filters=" + url.QueryEscape("Port=8080|SrcK8S_Namespace=test|DstK8S_Namespace=test"),
145145
outputQueries: []string{
146146
"?query={app=\"netobserv-flowcollector\",SrcK8S_Namespace=~\"(?i).*test.*\"}",
147147
"?query={app=\"netobserv-flowcollector\",DstK8S_Namespace=~\"(?i).*test.*\"}",
148-
"?query={app=\"netobserv-flowcollector\"}|~`Port\":8080,`",
148+
"?query={app=\"netobserv-flowcollector\"}|~`Port\":8080[,}]`",
149+
},
150+
}, {
151+
inputPath: "?filters=" + url.QueryEscape(`SrcK8S_Namespace=""&DstPort=70`),
152+
outputQueries: []string{
153+
"?query={app=\"netobserv-flowcollector\",SrcK8S_Namespace=\"\"}|~`DstPort\":70[,}]`",
154+
},
155+
}, {
156+
inputPath: "?filters=" + url.QueryEscape(`SrcK8S_Name=""&DstPort=70`),
157+
outputQueries: []string{
158+
"?query={app=\"netobserv-flowcollector\"}|~`DstPort\":70[,}]`|json|SrcK8S_Name=\"\"",
159+
},
160+
}, {
161+
inputPath: "?filters=" + url.QueryEscape(`SrcK8S_Name="",foo&DstK8S_Name="hello"`),
162+
outputQueries: []string{
163+
"?query={app=\"netobserv-flowcollector\"}|~`DstK8S_Name\":\"hello\"`|json|SrcK8S_Name=\"\"+or+SrcK8S_Name=~`(?i).*foo.*`",
164+
},
165+
}, {
166+
inputPath: "?filters=" + url.QueryEscape(`SrcK8S_Namespace=""|DstPort=70`),
167+
outputQueries: []string{
168+
"?query={app=\"netobserv-flowcollector\",SrcK8S_Namespace=\"\"}",
169+
"?query={app=\"netobserv-flowcollector\"}|~`DstPort\":70[,}]`",
170+
},
171+
}, {
172+
inputPath: "?filters=" + url.QueryEscape(`SrcK8S_Name=""|DstPort=70`),
173+
outputQueries: []string{
174+
"?query={app=\"netobserv-flowcollector\"}|~`DstPort\":70[,}]`",
175+
"?query={app=\"netobserv-flowcollector\"}|json|SrcK8S_Name=\"\"",
176+
},
177+
}, {
178+
inputPath: "?filters=" + url.QueryEscape(`SrcK8S_Name="",foo|DstK8S_Name="hello"`),
179+
outputQueries: []string{
180+
"?query={app=\"netobserv-flowcollector\"}|~`DstK8S_Name\":\"hello\"`",
181+
"?query={app=\"netobserv-flowcollector\"}|json|SrcK8S_Name=\"\"+or+SrcK8S_Name=~`(?i).*foo.*`",
182+
},
183+
}, {
184+
inputPath: "?filters=" + url.QueryEscape(`SrcK8S_Type="","Pod"`),
185+
outputQueries: []string{
186+
"?query={app=\"netobserv-flowcollector\"}|json|SrcK8S_Type=\"\"+or+SrcK8S_Type=\"Pod\"",
149187
},
150188
}}
151189

0 commit comments

Comments
 (0)