|
| 1 | +/* |
| 2 | + * Copyright (C) 2024 IBM, Inc. |
| 3 | + * |
| 4 | + * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | + * you may not use this file except in compliance with the License. |
| 6 | + * You may obtain a copy of the License at |
| 7 | + * |
| 8 | + * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | + * |
| 10 | + * Unless required by applicable law or agreed to in writing, software |
| 11 | + * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | + * See the License for the specific language governing permissions and |
| 14 | + * limitations under the License. |
| 15 | + * |
| 16 | + */ |
| 17 | + |
| 18 | +package transform |
| 19 | + |
| 20 | +import ( |
| 21 | + "fmt" |
| 22 | + "math" |
| 23 | + "strings" |
| 24 | + "sync" |
| 25 | + |
| 26 | + "github.com/netobserv/flowlogs-pipeline/pkg/api" |
| 27 | + "github.com/netobserv/flowlogs-pipeline/pkg/config" |
| 28 | + "github.com/netobserv/flowlogs-pipeline/pkg/operational" |
| 29 | + "github.com/netobserv/flowlogs-pipeline/pkg/utils" |
| 30 | + "github.com/sirupsen/logrus" |
| 31 | +) |
| 32 | + |
| 33 | +const ( |
| 34 | + defaultAnomalyWindow = 30 |
| 35 | + defaultAnomalySensitivity = 3.0 |
| 36 | +) |
| 37 | + |
| 38 | +var anomalyLog = logrus.WithField("component", "transform.Anomaly") |
| 39 | + |
| 40 | +type anomalyState struct { |
| 41 | + values []float64 |
| 42 | + sum float64 |
| 43 | + sumSq float64 |
| 44 | + baseline float64 |
| 45 | + initialized bool |
| 46 | +} |
| 47 | + |
| 48 | +func (s *anomalyState) addValue(v float64, window int) { |
| 49 | + s.values = append(s.values, v) |
| 50 | + s.sum += v |
| 51 | + s.sumSq += v * v |
| 52 | + if len(s.values) > window { |
| 53 | + oldest := s.values[0] |
| 54 | + s.values = s.values[1:] |
| 55 | + s.sum -= oldest |
| 56 | + s.sumSq -= oldest * oldest |
| 57 | + } |
| 58 | +} |
| 59 | + |
| 60 | +func (s *anomalyState) mean() float64 { |
| 61 | + if len(s.values) == 0 { |
| 62 | + return 0 |
| 63 | + } |
| 64 | + return s.sum / float64(len(s.values)) |
| 65 | +} |
| 66 | + |
| 67 | +func (s *anomalyState) stddev() float64 { |
| 68 | + if len(s.values) < 2 { |
| 69 | + return 0 |
| 70 | + } |
| 71 | + mean := s.mean() |
| 72 | + variance := (s.sumSq / float64(len(s.values))) - (mean * mean) |
| 73 | + if variance < 0 { |
| 74 | + variance = 0 |
| 75 | + } |
| 76 | + return math.Sqrt(variance) |
| 77 | +} |
| 78 | + |
| 79 | +type Anomaly struct { |
| 80 | + mu sync.Mutex |
| 81 | + states map[string]*anomalyState |
| 82 | + config api.TransformAnomaly |
| 83 | + windowSize int |
| 84 | + baselineWindow int |
| 85 | + sensitivity float64 |
| 86 | + alpha float64 |
| 87 | + opMetrics *operational.Metrics |
| 88 | +} |
| 89 | + |
| 90 | +// NewTransformAnomaly creates a new anomaly transformer. |
| 91 | +func NewTransformAnomaly(params config.StageParam, opMetrics *operational.Metrics) (Transformer, error) { |
| 92 | + anomalyConfig := api.TransformAnomaly{} |
| 93 | + if params.Transform != nil && params.Transform.Anomaly != nil { |
| 94 | + anomalyConfig = *params.Transform.Anomaly |
| 95 | + } |
| 96 | + if anomalyConfig.ValueField == "" { |
| 97 | + return nil, fmt.Errorf("valueField must be provided for anomaly transform") |
| 98 | + } |
| 99 | + |
| 100 | + window := anomalyConfig.WindowSize |
| 101 | + if window <= 0 { |
| 102 | + window = defaultAnomalyWindow |
| 103 | + } |
| 104 | + baselineWindow := anomalyConfig.BaselineWindow |
| 105 | + if baselineWindow <= 0 { |
| 106 | + baselineWindow = window / 2 |
| 107 | + if baselineWindow == 0 { |
| 108 | + baselineWindow = 1 |
| 109 | + } |
| 110 | + } |
| 111 | + sensitivity := anomalyConfig.Sensitivity |
| 112 | + if sensitivity <= 0 { |
| 113 | + sensitivity = defaultAnomalySensitivity |
| 114 | + } |
| 115 | + alpha := anomalyConfig.EWMAAlpha |
| 116 | + if alpha <= 0 { |
| 117 | + alpha = 2.0 / (float64(window) + 1.0) |
| 118 | + } |
| 119 | + if len(anomalyConfig.KeyFields) == 0 { |
| 120 | + anomalyConfig.KeyFields = []string{"SrcAddr", "DstAddr", "Proto"} |
| 121 | + } |
| 122 | + if anomalyConfig.Algorithm == "" { |
| 123 | + anomalyConfig.Algorithm = api.AnomalyAlgorithmZScore |
| 124 | + } |
| 125 | + |
| 126 | + anomalyLog.Infof("NewTransformAnomaly algorithm=%s window=%d baselineWindow=%d", anomalyConfig.Algorithm, window, baselineWindow) |
| 127 | + return &Anomaly{ |
| 128 | + states: make(map[string]*anomalyState), |
| 129 | + config: anomalyConfig, |
| 130 | + windowSize: window, |
| 131 | + baselineWindow: baselineWindow, |
| 132 | + sensitivity: sensitivity, |
| 133 | + alpha: alpha, |
| 134 | + opMetrics: opMetrics, |
| 135 | + }, nil |
| 136 | +} |
| 137 | + |
| 138 | +// Transform calculates anomaly scores per key and appends anomaly fields. |
| 139 | +func (a *Anomaly) Transform(entry config.GenericMap) (config.GenericMap, bool) { |
| 140 | + value, err := utils.ConvertToFloat64(entry[a.config.ValueField]) |
| 141 | + if err != nil { |
| 142 | + anomalyLog.Errorf("unable to convert %s to float: %v", a.config.ValueField, err) |
| 143 | + return entry, false |
| 144 | + } |
| 145 | + key := a.buildKey(entry) |
| 146 | + |
| 147 | + a.mu.Lock() |
| 148 | + state, ok := a.states[key] |
| 149 | + if !ok { |
| 150 | + state = &anomalyState{} |
| 151 | + a.states[key] = state |
| 152 | + } |
| 153 | + anomalyType, score := a.score(state, value) |
| 154 | + state.addValue(value, a.windowSize) |
| 155 | + stateSize := len(state.values) |
| 156 | + a.mu.Unlock() |
| 157 | + |
| 158 | + output := entry.Copy() |
| 159 | + output["anomaly_score"] = score |
| 160 | + output["anomaly_type"] = anomalyType |
| 161 | + output["baseline_window"] = stateSize |
| 162 | + |
| 163 | + return output, true |
| 164 | +} |
| 165 | + |
| 166 | +func (a *Anomaly) score(state *anomalyState, value float64) (string, float64) { |
| 167 | + if len(state.values) < a.baselineWindow { |
| 168 | + if !state.initialized { |
| 169 | + state.baseline = value |
| 170 | + state.initialized = true |
| 171 | + } |
| 172 | + return "warming_up", 0 |
| 173 | + } |
| 174 | + |
| 175 | + switch a.config.Algorithm { |
| 176 | + case api.AnomalyAlgorithmEWMA: |
| 177 | + return a.scoreEWMA(state, value) |
| 178 | + case api.AnomalyAlgorithmZScore: |
| 179 | + fallthrough |
| 180 | + default: |
| 181 | + return a.scoreZScore(state, value) |
| 182 | + } |
| 183 | +} |
| 184 | + |
| 185 | +func (a *Anomaly) scoreEWMA(state *anomalyState, value float64) (string, float64) { |
| 186 | + if !state.initialized { |
| 187 | + state.baseline = value |
| 188 | + state.initialized = true |
| 189 | + } |
| 190 | + deviation := value - state.baseline |
| 191 | + stddev := state.stddev() |
| 192 | + if stddev == 0 { |
| 193 | + stddev = math.Max(math.Abs(state.baseline)*1e-6, 1e-9) |
| 194 | + } |
| 195 | + score := math.Abs(deviation) / stddev |
| 196 | + state.baseline = state.baseline + a.alpha*(value-state.baseline) |
| 197 | + anomalyType := "normal" |
| 198 | + if score >= a.sensitivity { |
| 199 | + if deviation > 0 { |
| 200 | + anomalyType = "ewma_high" |
| 201 | + } else { |
| 202 | + anomalyType = "ewma_low" |
| 203 | + } |
| 204 | + } |
| 205 | + return anomalyType, score |
| 206 | +} |
| 207 | + |
| 208 | +func (a *Anomaly) scoreZScore(state *anomalyState, value float64) (string, float64) { |
| 209 | + mean := state.mean() |
| 210 | + stddev := state.stddev() |
| 211 | + if stddev == 0 { |
| 212 | + stddev = math.Max(math.Abs(mean)*1e-6, 1e-9) |
| 213 | + } |
| 214 | + score := math.Abs(value-mean) / stddev |
| 215 | + anomalyType := "normal" |
| 216 | + if score >= a.sensitivity { |
| 217 | + if value > mean { |
| 218 | + anomalyType = "zscore_high" |
| 219 | + } else { |
| 220 | + anomalyType = "zscore_low" |
| 221 | + } |
| 222 | + } |
| 223 | + return anomalyType, score |
| 224 | +} |
| 225 | + |
| 226 | +func (a *Anomaly) buildKey(entry config.GenericMap) string { |
| 227 | + parts := make([]string, 0, len(a.config.KeyFields)) |
| 228 | + for _, key := range a.config.KeyFields { |
| 229 | + if val, ok := entry[key]; ok { |
| 230 | + parts = append(parts, fmt.Sprint(val)) |
| 231 | + } else { |
| 232 | + parts = append(parts, "<missing>") |
| 233 | + } |
| 234 | + } |
| 235 | + return strings.Join(parts, "|") |
| 236 | +} |
| 237 | + |
| 238 | +// Reset clears the internal state; useful for tests. |
| 239 | +func (a *Anomaly) Reset() { |
| 240 | + a.mu.Lock() |
| 241 | + defer a.mu.Unlock() |
| 242 | + a.states = make(map[string]*anomalyState) |
| 243 | +} |
0 commit comments