Skip to content

Commit 0e8bdb3

Browse files
jotakjotak
committed
Manage open scales
1 parent 4991b24 commit 0e8bdb3

File tree

4 files changed

+47
-35
lines changed

4 files changed

+47
-35
lines changed

‎pkg/handler/alertingmock/alerting_mock.go‎

Lines changed: 17 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ func randomState() int {
6161
return firing
6262
}
6363

64-
func createAlert(probability float64, name, resourceName string, threshold int, targetLabels, resourceNames []string, annotations, labels model.LabelSet) (*Alert, int) {
64+
func createAlert(probability float64, name, resourceName string, threshold, upperBound int, targetLabels, resourceNames []string, annotations, labels model.LabelSet) (*Alert, int) {
6565
if rand.Float64() < probability {
6666
alertLabels := labels.Clone()
6767
alertState := randomState()
@@ -75,7 +75,7 @@ func createAlert(probability float64, name, resourceName string, threshold int,
7575
alertLabels[model.LabelName(lbl)] = model.LabelValue(resourceNames[idx])
7676
}
7777
}
78-
val := float64(threshold) + rand.Float64()*float64(100-threshold)
78+
val := float64(threshold) + rand.Float64()*float64(upperBound-threshold)
7979
return &Alert{
8080
Annotations: annotations,
8181
Labels: alertLabels,
@@ -86,19 +86,19 @@ func createAlert(probability float64, name, resourceName string, threshold int,
8686
return nil, 0
8787
}
8888

89-
func createAlerts(probability float64, name string, threshold int, targetLabels, resourceNames []string, annotations, labels model.LabelSet) ([]*Alert, int) {
89+
func createAlerts(probability float64, name string, threshold, upperBound int, targetLabels, resourceNames []string, annotations, labels model.LabelSet) ([]*Alert, int) {
9090
alerts := []*Alert{}
9191
var ruleState int
9292
for _, resourceName := range resourceNames {
93-
if alert, state := createAlert(probability, name, resourceName, threshold, targetLabels, resourceNames, annotations, labels); alert != nil {
93+
if alert, state := createAlert(probability, name, resourceName, threshold, upperBound, targetLabels, resourceNames, annotations, labels); alert != nil {
9494
ruleState |= state
9595
alerts = append(alerts, alert)
9696
}
9797
}
9898
return alerts, ruleState
9999
}
100100

101-
func createRule(probability float64, name, severity string, threshold int, bynetobs bool, nsLbl, nodeLbl []string) AlertingRule {
101+
func createRule(probability float64, name, severity string, threshold, upperBound int, bynetobs bool, nsLbl, nodeLbl []string) AlertingRule {
102102
labels := model.LabelSet{
103103
"severity": model.LabelValue(severity),
104104
}
@@ -126,23 +126,24 @@ func createRule(probability float64, name, severity string, threshold int, bynet
126126
jsonNodeLbl = fmt.Sprintf(`"nodeLabels":[%s],`, strings.Join(quotedLbl, ","))
127127
}
128128
annotations["netobserv_io_network_health"] = model.LabelValue(fmt.Sprintf(
129-
`{%s%s"threshold":"%d","unit":"%%"}`,
129+
`{%s%s"threshold":"%d","upperBound":"%d","unit":"%%"}`,
130130
jsonNsLbl,
131131
jsonNodeLbl,
132132
threshold,
133+
upperBound,
133134
))
134135
ruleLabels := labels.Clone()
135136
ruleLabels["prometheus"] = "openshift-monitoring/k8s"
136137
var alerts []*Alert
137138
var ruleState int
138139
if len(nsLbl) > 0 {
139-
alerts, ruleState = createAlerts(probability, name, threshold, nsLbl, namespaces, annotations, labels)
140+
alerts, ruleState = createAlerts(probability, name, threshold, upperBound, nsLbl, namespaces, annotations, labels)
140141
} else if len(nodeLbl) > 0 {
141-
alerts, ruleState = createAlerts(probability, name, threshold, nodeLbl, nodes, annotations, labels)
142+
alerts, ruleState = createAlerts(probability, name, threshold, upperBound, nodeLbl, nodes, annotations, labels)
142143
} else {
143144
// global
144145
alerts = []*Alert{}
145-
if alert, state := createAlert(probability, name, "", threshold, nil, nil, annotations, labels); alert != nil {
146+
if alert, state := createAlert(probability, name, "", threshold, upperBound, nil, nil, annotations, labels); alert != nil {
146147
ruleState |= state
147148
alerts = append(alerts, alert)
148149
}
@@ -159,12 +160,13 @@ func createRule(probability float64, name, severity string, threshold int, bynet
159160
func GetRules() func(w http.ResponseWriter, r *http.Request) {
160161
return func(w http.ResponseWriter, _ *http.Request) {
161162
alertingRules := []AlertingRule{
162-
createRule(0.4, "Packet delivery failed", "info", 5, true, []string{"SrcK8S_Namespace", "DstK8S_Namespace"}, []string{}),
163-
createRule(0.3, "You have reached your hourly rate limit", "info", 5, true, []string{"SrcK8S_Namespace", "DstK8S_Namespace"}, []string{}),
164-
createRule(0.1, "It's always DNS", "warning", 15, true, []string{"SrcK8S_Namespace", "DstK8S_Namespace"}, []string{}),
165-
createRule(0.1, "We're under attack", "warning", 20, true, []string{}, []string{}),
166-
createRule(0.1, "Sh*t - Famous last words", "critical", 5, true, []string{}, []string{"SrcK8S_Hostname", "DstK8S_Hostname"}),
167-
createRule(0.3, "FromIngress", "info", 10, false, []string{"exported_namespace"}, []string{}),
163+
createRule(0.4, "Packet delivery failed", "info", 5, 100, true, []string{"SrcK8S_Namespace", "DstK8S_Namespace"}, []string{}),
164+
createRule(0.3, "You have reached your hourly rate limit", "info", 5, 100, true, []string{"SrcK8S_Namespace", "DstK8S_Namespace"}, []string{}),
165+
createRule(0.1, "It's always DNS", "warning", 15, 100, true, []string{"SrcK8S_Namespace", "DstK8S_Namespace"}, []string{}),
166+
createRule(0.1, "We're under attack", "warning", 20, 100, true, []string{}, []string{}),
167+
createRule(0.1, "Sh*t - Famous last words", "critical", 5, 100, true, []string{}, []string{"SrcK8S_Hostname", "DstK8S_Hostname"}),
168+
createRule(0.3, "FromIngress", "info", 10, 100, false, []string{"exported_namespace"}, []string{}),
169+
createRule(0.3, "Degraded latency", "info", 100, 1000, true, []string{"SrcK8S_Namespace", "DstK8S_Namespace"}, []string{}),
168170
}
169171
res := map[string]any{
170172
"status": "success",

‎web/src/components/health/__tests__/helper.spec.tsx‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ const mockAlert = (
1414
state: state as AlertStates,
1515
annotations: {},
1616
ruleID: '',
17-
metadata: { thresholdF: threshold, threshold: '', unit: '%' },
17+
metadata: { thresholdF: threshold, threshold: '', upperBoundF: 100, upperBound: '', unit: '%' },
1818
value: value
1919
};
2020
};
@@ -41,6 +41,26 @@ describe('health helpers', () => {
4141
expect(score.weight).toEqual(0.075);
4242
});
4343

44+
it('should compute unweighted alert score with upper bound', () => {
45+
const alert = mockAlert('test', 'critical', 'firing', 100, 500);
46+
alert.metadata!.upperBoundF = 1000;
47+
const score = computeAlertScore(alert);
48+
expect(score.rawScore).toBeCloseTo(5.26, 2);
49+
});
50+
51+
it('should compute unweighted alert score with clamping', () => {
52+
// below threshold
53+
const alert = mockAlert('test', 'critical', 'firing', 100, 1);
54+
alert.metadata!.upperBoundF = 1000;
55+
let score = computeAlertScore(alert);
56+
expect(score.rawScore).toEqual(10);
57+
58+
// above upper bound
59+
alert.value = 5000;
60+
score = computeAlertScore(alert);
61+
expect(score.rawScore).toEqual(0);
62+
});
63+
4464
it('should compute full score', () => {
4565
// Start with an empty one => max score
4666
const r: ByResource = {

‎web/src/components/health/health-color-square.tsx‎

Lines changed: 1 addition & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -60,20 +60,6 @@ const buildGradientCSS = (colorMap: ColorMap): string => {
6060
export const HealthColorSquare: React.FC<HealthColorSquareProps> = ({ alert }) => {
6161
const { t } = useTranslation('plugin__netobserv-plugin');
6262

63-
let prefix = '';
64-
switch (alert.state) {
65-
case 'pending':
66-
prefix = `[pending] `;
67-
break;
68-
case 'silenced':
69-
prefix = `[silenced] `;
70-
break;
71-
}
72-
const valueInfo =
73-
valueFormat(alert.value as number, 2) +
74-
(alert.metadata?.threshold ? '> ' + alert.metadata.threshold + ' ' + alert.metadata.unit : '');
75-
const tooltip = `${prefix}${alert.annotations['summary']} | ${valueInfo}`;
76-
7763
const colorMap =
7864
alert.labels.severity === 'critical'
7965
? criticalColorMap
@@ -98,7 +84,7 @@ export const HealthColorSquare: React.FC<HealthColorSquareProps> = ({ alert }) =
9884
</>
9985
}
10086
>
101-
<div className={'cell'} style={getCellColors(scoreForMap, 0, 1, colorMap)} title={tooltip} />
87+
<div className={'cell'} style={getCellColors(scoreForMap, 0, 1, colorMap)} />
10288
</Tooltip>
10389
);
10490
};

‎web/src/components/health/helper.ts‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,8 @@ type RuleWithMetadata = Rule & {
3636
type HealthMetadata = {
3737
threshold: string;
3838
thresholdF: number;
39+
upperBound: string;
40+
upperBoundF: number;
3941
unit: string;
4042
nodeLabels?: string[];
4143
namespaceLabels?: string[];
@@ -51,6 +53,7 @@ export const getHealthMetadata = (annotations: PrometheusLabels): HealthMetadata
5153
const md = (JSON.parse(annotations['netobserv_io_network_health']) as HealthMetadata) || undefined;
5254
if (md) {
5355
md.thresholdF = parseFloat(md.threshold) || 0;
56+
md.upperBoundF = parseFloat(md.upperBound) || 0;
5457
}
5558
return md;
5659
}
@@ -274,11 +277,12 @@ export const computeScore = (r: ByResource): number => {
274277

275278
// Score [0,1]; lower is better
276279
const computeExcessRatio = (a: AlertWithRuleName): number => {
277-
// Assuming the alert value is a [0-100] percentage. Needs update if more use cases come up.
280+
// Assuming the alert value is a [0-n] percentage. Needs update if more use cases come up.
278281
const threshold = (a.metadata?.thresholdF || 0) / 2;
279-
const range = 100 - threshold;
280-
const excess = Math.max((a.value as number) - threshold, 0);
281-
return excess / range;
282+
const upper = a.metadata?.upperBoundF || 100;
283+
const vclamped = Math.min(Math.max(a.value as number, threshold), upper);
284+
const range = upper - threshold;
285+
return (vclamped - threshold) / range;
282286
};
283287

284288
export const computeExcessRatioStatusWeighted = (a: AlertWithRuleName): number => {

0 commit comments

Comments
 (0)