Skip to content

Commit 7b5ba50

Browse files
jotakjotak
committed
NETOBSERV-1990: fix NOT filters with multiple values
NOT filters with multiple values should generate AND queries, not OR E.g. IP=10.0.0.1,10.0.0.2 generates IP=10.0.0.1 OR IP=10.0.0.2, which is correct But, IP!=10.0.0.1,10.0.0.2 should not generate IP!=10.0.0.1 OR IP!=10.0.0.2 (which would be always true) but IP!=10.0.0.1 AND IP!=10.0.0.2 instead Reasoning with CIDRs is the same: IP!=10.0.0.0/8,172.0.0.0/8 translates into IP NOT IN 10.0.0.0/8 AND IP NOT IN 172.0.0.0/8
1 parent 01883f0 commit 7b5ba50

File tree

6 files changed

+137
-64
lines changed

6 files changed

+137
-64
lines changed

‎.golangci.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,5 +26,8 @@ linters-settings:
2626
- rangeValCopy
2727
- indexAlloc
2828
- deprecatedComment
29+
settings:
30+
ifElseChain:
31+
minThreshold: 3
2932
cyclop:
3033
max-complexity: 20

‎pkg/loki/flow_query.go‎

Lines changed: 15 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -156,21 +156,26 @@ func (q *FlowQueryBuilder) addLineFilters(key string, values []string, not bool,
156156
// addIPFilters assumes that we are searching for that IP addresses as part
157157
// of the log line (not in the stream selector labels)
158158
func (q *FlowQueryBuilder) addIPFilters(key string, values []string, not bool) {
159+
if not {
160+
// NOT IP filters means we don't want any of the values, ie. IP!=A AND IP!=B instead of IP!=A OR IP!=B
161+
for _, value := range values {
162+
// empty exact matches should be treated as attribute filters looking for empty IP
163+
if value == emptyMatch {
164+
q.jsonFilters = append(q.jsonFilters, []filters.LabelFilter{filters.NotStringLabelFilter(key, "")})
165+
} else {
166+
q.jsonFilters = append(q.jsonFilters, []filters.LabelFilter{filters.NotIPLabelFilter(key, value)})
167+
}
168+
}
169+
return
170+
}
171+
// Positive case
159172
filtersPerKey := make([]filters.LabelFilter, 0, len(values))
160173
for _, value := range values {
161174
// empty exact matches should be treated as attribute filters looking for empty IP
162175
if value == emptyMatch {
163-
if not {
164-
filtersPerKey = append(filtersPerKey, filters.NotStringLabelFilter(key, ""))
165-
} else {
166-
filtersPerKey = append(filtersPerKey, filters.StringEqualLabelFilter(key, ""))
167-
}
176+
filtersPerKey = append(filtersPerKey, filters.StringEqualLabelFilter(key, ""))
168177
} else {
169-
if not {
170-
filtersPerKey = append(filtersPerKey, filters.NotIPLabelFilter(key, value))
171-
} else {
172-
filtersPerKey = append(filtersPerKey, filters.IPLabelFilter(key, value))
173-
}
178+
filtersPerKey = append(filtersPerKey, filters.IPLabelFilter(key, value))
174179
}
175180
}
176181
q.jsonFilters = append(q.jsonFilters, filtersPerKey)

‎pkg/model/filters/logql.go‎

Lines changed: 82 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -252,6 +252,7 @@ func NotContainsKeyLineFilter(key string) LineFilter {
252252
}
253253
}
254254

255+
// NumericLineFilter returns a LineFilter and true if it has an empty match
255256
func NumericLineFilter(key string, values []string, not, moreThan bool) (LineFilter, bool) {
256257
return checkExact(
257258
LineFilter{
@@ -271,6 +272,7 @@ func ArrayLineFilter(key string, values []string, not bool) LineFilter {
271272
return lf
272273
}
273274

275+
// StringLineFilterCheckExact returns a LineFilter and true if it has an empty match
274276
func StringLineFilterCheckExact(key string, values []string, not bool) (LineFilter, bool) {
275277
return checkExact(LineFilter{key: key, not: not}, values, typeRegexContains)
276278
}
@@ -364,21 +366,11 @@ func moreThanRegex(sb *strings.Builder, value string) {
364366
// under construction (contained in the provided strings.Builder)
365367
func (f *LineFilter) WriteInto(sb *strings.Builder) {
366368
if f.not {
367-
if !f.allowEmpty {
368-
// the record must contains the field if values are specified
369-
// since FLP skip empty fields / zeros values
370-
if len(f.values) > 0 {
371-
sb.WriteString("|~`\"")
372-
sb.WriteString(f.key)
373-
sb.WriteString("\"`")
374-
}
375-
}
376-
// then we exclude match results
377-
sb.WriteString("!~`")
378-
} else {
379-
sb.WriteString("|~`")
369+
f.writeIntoNot(sb)
370+
return
380371
}
381372

373+
sb.WriteString("|~`")
382374
if len(f.values) == 0 {
383375
// match only the end of KEY if not 'strictKey'
384376
// no value will be provided here as we only check if key exists
@@ -392,49 +384,85 @@ func (f *LineFilter) WriteInto(sb *strings.Builder) {
392384
if i > 0 {
393385
sb.WriteByte('|')
394386
}
387+
f.writeValueInto(sb, v)
388+
}
389+
}
390+
sb.WriteRune('`')
391+
}
395392

396-
// match only the end of KEY + regex VALUE if not 'strictKey'
397-
// if numeric, KEY":VALUE,
398-
// if string KEY":"VALUE"
399-
// ie 'Port' key will match both 'SrcPort":"XXX"' and 'DstPort":"XXX"
400-
// VALUE can be quoted for exact match or contains * to inject regex any
401-
// For numeric values, exact match is implicit
402-
// (the trick is to match for the ending coma; it works as long as the filtered field
403-
// is not the last one (they're in alphabetic order); a less performant alternative
404-
// but more future-proof/less hacky could be to move that to a json filter, if needed)
405-
if f.strictKey {
406-
sb.WriteByte('"')
407-
}
393+
// WriteInto transforms a LineFilter to its corresponding part of a LogQL query
394+
// under construction (contained in the provided strings.Builder)
395+
func (f *LineFilter) writeIntoNot(sb *strings.Builder) {
396+
if !f.allowEmpty {
397+
// the record must contains the field if values are specified
398+
// since FLP skip empty fields / zeros values
399+
if len(f.values) > 0 {
400+
sb.WriteString("|~`\"")
408401
sb.WriteString(f.key)
409-
sb.WriteString(`":`)
410-
switch v.valueType {
411-
case typeNumber, typeRegex:
412-
if f.moreThan {
413-
moreThanRegex(sb, v.value)
414-
} else {
415-
sb.WriteString(v.value)
416-
}
417-
// a number or regex can be followed by } if it's the last property of a JSON document
418-
sb.WriteString("[,}]")
419-
case typeBool:
420-
sb.WriteString(v.value)
421-
case typeString, typeIP:
422-
// exact matches are specified as just strings
423-
sb.WriteByte('"')
424-
sb.WriteString(valueReplacer.Replace(v.value))
425-
sb.WriteByte('"')
426-
// contains-match are specified as regular expressions
427-
case typeRegexContains:
428-
sb.WriteString(`"(?i)[^"]*`)
429-
sb.WriteString(valueReplacer.Replace(v.value))
430-
sb.WriteString(`.*"`)
431-
// for array, we ensure it starts by [ and ends by ]
432-
case typeRegexArrayContains:
433-
sb.WriteString(`\[(?i)[^]]*`)
434-
sb.WriteString(valueReplacer.Replace(v.value))
435-
sb.WriteString(`[^]]*]`)
436-
}
402+
sb.WriteString("\"`")
437403
}
438404
}
439-
sb.WriteRune('`')
405+
406+
if len(f.values) == 0 {
407+
// then we exclude match results
408+
sb.WriteString("!~`")
409+
410+
// match only the end of KEY if not 'strictKey'
411+
// no value will be provided here as we only check if key exists
412+
if f.strictKey {
413+
sb.WriteByte('"')
414+
}
415+
sb.WriteString(f.key)
416+
sb.WriteString("\"`")
417+
} else {
418+
for _, v := range f.values {
419+
sb.WriteString("!~`")
420+
f.writeValueInto(sb, v)
421+
sb.WriteRune('`')
422+
}
423+
}
424+
}
425+
426+
func (f *LineFilter) writeValueInto(sb *strings.Builder, v lineMatch) {
427+
// match only the end of KEY + regex VALUE if not 'strictKey'
428+
// if numeric, KEY":VALUE,
429+
// if string KEY":"VALUE"
430+
// ie 'Port' key will match both 'SrcPort":"XXX"' and 'DstPort":"XXX"
431+
// VALUE can be quoted for exact match or contains * to inject regex any
432+
// For numeric values, exact match is implicit
433+
// (the trick is to match for the ending coma; it works as long as the filtered field
434+
// is not the last one (they're in alphabetic order); a less performant alternative
435+
// but more future-proof/less hacky could be to move that to a json filter, if needed)
436+
if f.strictKey {
437+
sb.WriteByte('"')
438+
}
439+
sb.WriteString(f.key)
440+
sb.WriteString(`":`)
441+
switch v.valueType {
442+
case typeNumber, typeRegex:
443+
if f.moreThan {
444+
moreThanRegex(sb, v.value)
445+
} else {
446+
sb.WriteString(v.value)
447+
}
448+
// a number or regex can be followed by } if it's the last property of a JSON document
449+
sb.WriteString("[,}]")
450+
case typeBool:
451+
sb.WriteString(v.value)
452+
case typeString, typeIP:
453+
// exact matches are specified as just strings
454+
sb.WriteByte('"')
455+
sb.WriteString(valueReplacer.Replace(v.value))
456+
sb.WriteByte('"')
457+
// contains-match are specified as regular expressions
458+
case typeRegexContains:
459+
sb.WriteString(`"(?i)[^"]*`)
460+
sb.WriteString(valueReplacer.Replace(v.value))
461+
sb.WriteString(`.*"`)
462+
// for array, we ensure it starts by [ and ends by ]
463+
case typeRegexArrayContains:
464+
sb.WriteString(`\[(?i)[^]]*`)
465+
sb.WriteString(valueReplacer.Replace(v.value))
466+
sb.WriteString(`[^]]*]`)
467+
}
440468
}

‎pkg/model/filters/logql_test.go‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,3 +74,22 @@ func TestWriteInto_7654(t *testing.T) {
7474
assert.Equal(t, reg.MatchString(val), i >= 7654, fmt.Sprintf("Value: %d", i))
7575
}
7676
}
77+
78+
func TestMultiStrings(t *testing.T) {
79+
lf, ok := StringLineFilterCheckExact("foo", []string{`"a"`, `"b"`}, false)
80+
assert.False(t, ok)
81+
sb := strings.Builder{}
82+
lf.WriteInto(&sb)
83+
assert.Equal(t, "|~"+backtick(`foo":"a"|foo":"b"`), sb.String())
84+
85+
// Repeat with "not" (here we expect foo being neither a nor b)
86+
lf, ok = StringLineFilterCheckExact("foo", []string{`"a"`, `"b"`}, true)
87+
assert.False(t, ok)
88+
sb = strings.Builder{}
89+
lf.WriteInto(&sb)
90+
assert.Equal(t, "|~"+backtick(`"foo"`)+"!~"+backtick(`foo":"a"`)+"!~"+backtick(`foo":"b"`), sb.String())
91+
}
92+
93+
func backtick(str string) string {
94+
return "`" + str + "`"
95+
}

‎pkg/server/server_flows_test.go‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,12 @@ func TestLokiFiltering(t *testing.T) {
6464
outputQueries: []string{
6565
"?query={app=\"netobserv-flowcollector\"}|~`SrcK8S_Name\":\"(?i)[^\"]*name1.*\"|SrcK8S_Name\":\"(?i)[^\"]*name2.*\"`",
6666
},
67+
}, {
68+
name: "NOT line filter same key",
69+
inputPath: "?filters=" + url.QueryEscape(`SrcK8S_Name!="name1","name2"`),
70+
outputQueries: []string{
71+
"?query={app=\"netobserv-flowcollector\"}|~`\"SrcK8S_Name\"`!~`SrcK8S_Name\":\"name1\"`!~`SrcK8S_Name\":\"name2\"`",
72+
},
6773
}, {
6874
name: "OR label filter same key",
6975
inputPath: "?filters=" + url.QueryEscape("SrcK8S_Namespace=ns1,ns2"),
@@ -90,6 +96,12 @@ func TestLokiFiltering(t *testing.T) {
9096
outputQueries: []string{
9197
"?query={app=\"netobserv-flowcollector\"}|json|SrcAddr=ip(\"10.128.0.1\")+or+SrcAddr=ip(\"10.128.0.2\")",
9298
},
99+
}, {
100+
name: "NOT IP filters",
101+
inputPath: "?filters=" + url.QueryEscape(`SrcAddr!=10.128.0.1,10.128.0.2`),
102+
outputQueries: []string{
103+
"?query={app=\"netobserv-flowcollector\"}|json|SrcAddr!=ip(\"10.128.0.1\")|SrcAddr!=ip(\"10.128.0.2\")",
104+
},
93105
}, {
94106
name: "Several OR filters",
95107
inputPath: "?filters=" + url.QueryEscape("SrcPort=8080|SrcAddr=10.128.0.1|SrcK8S_Namespace=default"),

‎web/src/model/__tests__/filters.spec.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { FilterDefinitionSample } from '../../components/__tests-data__/filters';
22
import { findFilter } from '../../utils/filter-definitions';
33
import { doesIncludeFilter, Filter, filtersEqual } from '../filters';
4+
import { filtersToString } from '../flow-query';
45

56
describe('doesIncludeFilter', () => {
67
const srcNameFilter = findFilter(FilterDefinitionSample, 'src_name')!;
@@ -18,6 +19,11 @@ describe('doesIncludeFilter', () => {
1819
}
1920
];
2021

22+
it('should encode as', () => {
23+
const asString = filtersToString(activeFilters, false);
24+
expect(asString).toEqual(encodeURIComponent('SrcK8S_Name=abc,def&DstK8S_Name!=abc,def'));
25+
});
26+
2127
it('should not include filter due to different key', () => {
2228
const isIncluded = doesIncludeFilter(activeFilters, { def: findFilter(FilterDefinitionSample, 'protocol')! }, [
2329
{ v: 'abc' },

0 commit comments

Comments
 (0)