Skip to content

Commit 76131e0

Browse files
jotakjotak
committed
Share filter predicate code (prom+transform)
Move prom-encode predicates filtering code to its own package and share it with "keep_entry" transforms
1 parent 417769d commit 76131e0

File tree

7 files changed

+341
-218
lines changed

7 files changed

+341
-218
lines changed

‎pkg/api/transform_filter.go‎

Lines changed: 12 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -17,23 +17,14 @@
1717

1818
package api
1919

20-
import (
21-
"errors"
22-
"regexp"
23-
)
24-
2520
type TransformFilter struct {
2621
Rules []TransformFilterRule `yaml:"rules,omitempty" json:"rules,omitempty" doc:"list of filter rules, each includes:"`
2722
}
2823

29-
func (tf *TransformFilter) Preprocess() error {
30-
var errs []error
24+
func (tf *TransformFilter) Preprocess() {
3125
for i := range tf.Rules {
32-
if err := tf.Rules[i].preprocess(); err != nil {
33-
errs = append(errs, err)
34-
}
26+
tf.Rules[i].preprocess()
3527
}
36-
return errors.Join(errs...)
3728
}
3829

3930
type TransformFilterEnum string
@@ -92,35 +83,22 @@ type TransformFilterRule struct {
9283
ConditionalSampling []*SamplingCondition `yaml:"conditionalSampling,omitempty" json:"conditionalSampling,omitempty" doc:"sampling configuration rules"`
9384
}
9485

95-
func (r *TransformFilterRule) preprocess() error {
96-
var errs []error
86+
func (r *TransformFilterRule) preprocess() {
9787
if r.RemoveField != nil {
98-
if err := r.RemoveField.preprocess(false); err != nil {
99-
errs = append(errs, err)
100-
}
88+
r.RemoveField.preprocess()
10189
}
10290
if r.RemoveEntry != nil {
103-
if err := r.RemoveEntry.preprocess(false); err != nil {
104-
errs = append(errs, err)
105-
}
91+
r.RemoveEntry.preprocess()
10692
}
10793
for i := range r.RemoveEntryAllSatisfied {
108-
if err := r.RemoveEntryAllSatisfied[i].RemoveEntry.preprocess(false); err != nil {
109-
errs = append(errs, err)
110-
}
94+
r.RemoveEntryAllSatisfied[i].RemoveEntry.preprocess()
11195
}
11296
for i := range r.KeepEntryAllSatisfied {
113-
err := r.KeepEntryAllSatisfied[i].KeepEntry.preprocess(r.KeepEntryAllSatisfied[i].Type == KeepEntryIfRegexMatch || r.KeepEntryAllSatisfied[i].Type == KeepEntryIfNotRegexMatch)
114-
if err != nil {
115-
errs = append(errs, err)
116-
}
97+
r.KeepEntryAllSatisfied[i].KeepEntry.preprocess()
11798
}
11899
for i := range r.ConditionalSampling {
119-
if err := r.ConditionalSampling[i].preprocess(); err != nil {
120-
errs = append(errs, err)
121-
}
100+
r.ConditionalSampling[i].preprocess()
122101
}
123-
return errors.Join(errs...)
124102
}
125103

126104
type TransformFilterGenericRule struct {
@@ -129,25 +107,12 @@ type TransformFilterGenericRule struct {
129107
CastInt bool `yaml:"castInt,omitempty" json:"castInt,omitempty" doc:"set true to cast the value field as an int (numeric values are float64 otherwise)"`
130108
}
131109

132-
func (r *TransformFilterGenericRule) preprocess(isRegex bool) error {
133-
if isRegex {
134-
if s, ok := r.Value.(string); ok {
135-
v, err := regexp.Compile(s)
136-
if err != nil {
137-
r.Value = nil
138-
return err
139-
}
140-
r.Value = v
141-
} else {
142-
r.Value = nil
143-
return errors.New("regex filter expects string value")
144-
}
145-
} else if r.CastInt {
110+
func (r *TransformFilterGenericRule) preprocess() {
111+
if r.CastInt {
146112
if f, ok := r.Value.(float64); ok {
147113
r.Value = int(f)
148114
}
149115
}
150-
return nil
151116
}
152117

153118
type TransformFilterRuleWithAssignee struct {
@@ -172,12 +137,8 @@ type SamplingCondition struct {
172137
Rules []*RemoveEntryRule `yaml:"rules,omitempty" json:"rules,omitempty" doc:"rules to be satisfied for this sampling configuration"`
173138
}
174139

175-
func (s *SamplingCondition) preprocess() error {
176-
var errs []error
140+
func (s *SamplingCondition) preprocess() {
177141
for i := range s.Rules {
178-
if err := s.Rules[i].RemoveEntry.preprocess(false); err != nil {
179-
errs = append(errs, err)
180-
}
142+
s.Rules[i].RemoveEntry.preprocess()
181143
}
182-
return errors.Join(errs...)
183144
}

‎pkg/pipeline/encode/encode_prom_metric.go‎

Lines changed: 12 additions & 97 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,15 @@
11
package encode
22

33
import (
4-
"fmt"
54
"regexp"
6-
"strings"
75

86
"github.com/netobserv/flowlogs-pipeline/pkg/api"
9-
"github.com/netobserv/flowlogs-pipeline/pkg/config"
7+
"github.com/netobserv/flowlogs-pipeline/pkg/utils/filters"
108
)
119

12-
type Predicate func(flow config.GenericMap) bool
13-
14-
var variableExtractor = regexp.MustCompile(`\$\(([^\)]+)\)`)
15-
1610
type MetricInfo struct {
1711
*api.MetricsItem
18-
FilterPredicates []Predicate
12+
FilterPredicates []filters.Predicate
1913
MappedLabels []MappedLabel
2014
}
2115

@@ -32,104 +26,25 @@ func (m *MetricInfo) TargetLabels() []string {
3226
return targetLabels
3327
}
3428

35-
func Presence(filter api.MetricsFilter) Predicate {
36-
return func(flow config.GenericMap) bool {
37-
_, found := flow[filter.Key]
38-
return found
39-
}
40-
}
41-
42-
func Absence(filter api.MetricsFilter) Predicate {
43-
return func(flow config.GenericMap) bool {
44-
_, found := flow[filter.Key]
45-
return !found
46-
}
47-
}
48-
49-
func Equal(filter api.MetricsFilter) Predicate {
50-
varLookups := extractVarLookups(filter.Value)
51-
return func(flow config.GenericMap) bool {
52-
if val, found := flow[filter.Key]; found {
53-
sVal, ok := val.(string)
54-
if !ok {
55-
sVal = fmt.Sprint(val)
56-
}
57-
value := filter.Value
58-
if len(varLookups) > 0 {
59-
value = injectVars(flow, value, varLookups)
60-
}
61-
return sVal == value
62-
}
63-
return false
64-
}
65-
}
66-
67-
func NotEqual(filter api.MetricsFilter) Predicate {
68-
pred := Equal(filter)
69-
return func(flow config.GenericMap) bool { return !pred(flow) }
70-
}
71-
72-
func Regex(filter api.MetricsFilter) Predicate {
73-
r, _ := regexp.Compile(filter.Value)
74-
return func(flow config.GenericMap) bool {
75-
if val, found := flow[filter.Key]; found {
76-
sVal, ok := val.(string)
77-
if !ok {
78-
sVal = fmt.Sprint(val)
79-
}
80-
return r.MatchString(sVal)
81-
}
82-
return false
83-
}
84-
}
85-
86-
func NotRegex(filter api.MetricsFilter) Predicate {
87-
pred := Regex(filter)
88-
return func(flow config.GenericMap) bool { return !pred(flow) }
89-
}
90-
91-
func filterToPredicate(filter api.MetricsFilter) Predicate {
29+
func filterToPredicate(filter api.MetricsFilter) filters.Predicate {
9230
switch filter.Type {
9331
case api.MetricFilterEqual:
94-
return Equal(filter)
32+
return filters.Equal(filter.Key, filter.Value, true)
9533
case api.MetricFilterNotEqual:
96-
return NotEqual(filter)
34+
return filters.NotEqual(filter.Key, filter.Value, true)
9735
case api.MetricFilterPresence:
98-
return Presence(filter)
36+
return filters.Presence(filter.Key)
9937
case api.MetricFilterAbsence:
100-
return Absence(filter)
38+
return filters.Absence(filter.Key)
10139
case api.MetricFilterRegex:
102-
return Regex(filter)
40+
r, _ := regexp.Compile(filter.Value)
41+
return filters.Regex(filter.Key, r)
10342
case api.MetricFilterNotRegex:
104-
return NotRegex(filter)
43+
r, _ := regexp.Compile(filter.Value)
44+
return filters.NotRegex(filter.Key, r)
10545
}
10646
// Default = Exact
107-
return Equal(filter)
108-
}
109-
110-
func extractVarLookups(value string) [][]string {
111-
// Extract list of variables to lookup
112-
// E.g: filter "$(SrcAddr):$(SrcPort)" would return [SrcAddr,SrcPort]
113-
if len(value) > 0 {
114-
return variableExtractor.FindAllStringSubmatch(value, -1)
115-
}
116-
return nil
117-
}
118-
119-
func injectVars(flow config.GenericMap, filterValue string, varLookups [][]string) string {
120-
injected := filterValue
121-
for _, matchGroup := range varLookups {
122-
var value string
123-
if rawVal, found := flow[matchGroup[1]]; found {
124-
if sVal, ok := rawVal.(string); ok {
125-
value = sVal
126-
} else {
127-
value = fmt.Sprint(rawVal)
128-
}
129-
}
130-
injected = strings.ReplaceAll(injected, matchGroup[0], value)
131-
}
132-
return injected
47+
return filters.Equal(filter.Key, filter.Value, true)
13348
}
13449

13550
func CreateMetricInfo(def *api.MetricsItem) *MetricInfo {

‎pkg/pipeline/encode/encode_prom_test.go‎

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -754,12 +754,3 @@ func Test_MultipleProm(t *testing.T) {
754754

755755
// TODO: Add test for different addresses, but need to deal with StartPromServer (ListenAndServe)
756756
}
757-
758-
func Test_Filters_extractVarLookups(t *testing.T) {
759-
variables := extractVarLookups("$(abc)--$(def)")
760-
761-
require.Equal(t, [][]string{{"$(abc)", "abc"}, {"$(def)", "def"}}, variables)
762-
763-
variables = extractVarLookups("")
764-
require.Empty(t, variables)
765-
}

0 commit comments

Comments
 (0)