Skip to content

Commit d822569

Browse files
JiaweiWukibanamachineelasticmachine
authored
[ResponseOps] Clear flapping when rules are enabled or bulk enabled (#235024)
## Summary Resolves: #149950 Clears flapping on alerts generated by rules when the user enables or bulk enables the rule when it is disabled. Should only clear flapping on active or recovered alerts, meaning untracked alerts are untouched. ### To test: 1. Ensure flapping is enabled 2. Create a rule(s) and generate a few alerts 3. Disable the rule(s) 4. Assert the alerts have a flapping history 5. Enable the rule(s) 6. Assert the previously generated active or recovered alerts now have flapping: false and flapping history is cleared 7. Assert that this only works on lifecycle rules (not siem rules) ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --------- Co-authored-by: kibanamachine <[email protected]> Co-authored-by: Elastic Machine <[email protected]>
1 parent acb3a27 commit d822569

File tree

11 files changed

+862
-5
lines changed

11 files changed

+862
-5
lines changed

‎x-pack/platform/plugins/shared/alerting/server/alerts_service/alerts_service.mock.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ const creatAlertsServiceMock = () => {
1313
getContextInitializationPromise: jest.fn(),
1414
createAlertsClient: jest.fn(),
1515
setAlertsToUntracked: jest.fn(),
16+
clearAlertFlappingHistory: jest.fn(),
1617
};
1718
});
1819
};

‎x-pack/platform/plugins/shared/alerting/server/alerts_service/alerts_service.ts‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,8 @@ import { AlertsClient } from '../alerts_client';
5050
import type { IAlertsClient } from '../alerts_client/types';
5151
import type { SetAlertsToUntrackedParams } from './lib/set_alerts_to_untracked';
5252
import { setAlertsToUntracked } from './lib/set_alerts_to_untracked';
53+
import type { ClearAlertFlappingHistoryParams } from './lib/clear_alert_flapping_history';
54+
import { clearAlertFlappingHistory } from './lib/clear_alert_flapping_history';
5355

5456
export const TOTAL_FIELDS_LIMIT = 2500;
5557
const LEGACY_ALERT_CONTEXT = 'legacy-alert';
@@ -501,4 +503,12 @@ export class AlertsService implements IAlertsService {
501503
...opts,
502504
});
503505
}
506+
507+
public async clearAlertFlappingHistory(opts: ClearAlertFlappingHistoryParams) {
508+
return clearAlertFlappingHistory({
509+
logger: this.options.logger,
510+
esClient: await this.options.elasticsearchClientPromise,
511+
...opts,
512+
});
513+
}
504514
}

‎x-pack/platform/plugins/shared/alerting/server/alerts_service/lib/clear_alert_flapping_history.test.ts‎

Lines changed: 183 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,183 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
import type { ElasticsearchClientMock } from '@kbn/core/server/mocks';
9+
import { elasticsearchServiceMock, loggingSystemMock } from '@kbn/core/server/mocks';
10+
import { clearAlertFlappingHistory } from './clear_alert_flapping_history';
11+
import { ALERT_UUID } from '@kbn/rule-data-utils';
12+
13+
let clusterClient: ElasticsearchClientMock;
14+
let logger: ReturnType<(typeof loggingSystemMock)['createLogger']>;
15+
16+
describe('clearAlertFlappingHistory()', () => {
17+
beforeEach(() => {
18+
logger = loggingSystemMock.createLogger();
19+
clusterClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
20+
clusterClient.updateByQuery.mockResponse({
21+
total: 1,
22+
updated: 1,
23+
});
24+
25+
clusterClient.search.mockResponse({
26+
took: 1,
27+
timed_out: false,
28+
_shards: {
29+
total: 1,
30+
successful: 1,
31+
skipped: 0,
32+
failed: 0,
33+
},
34+
hits: {
35+
hits: [
36+
{
37+
_index: 'test-index',
38+
_source: {
39+
[ALERT_UUID]: 'test-alert-id',
40+
},
41+
},
42+
],
43+
},
44+
});
45+
});
46+
47+
afterEach(() => {
48+
jest.resetAllMocks();
49+
});
50+
51+
test('should call updateByQuery with provided rule id and index', async () => {
52+
await clearAlertFlappingHistory({
53+
logger,
54+
esClient: clusterClient,
55+
indices: ['test-index'],
56+
ruleIds: ['test-rule'],
57+
});
58+
59+
expect(clusterClient.updateByQuery).toHaveBeenCalledTimes(1);
60+
expect(clusterClient.updateByQuery.mock.lastCall).toMatchInlineSnapshot(`
61+
Array [
62+
Object {
63+
"allow_no_indices": true,
64+
"conflicts": "proceed",
65+
"index": Array [
66+
"test-index",
67+
],
68+
"query": Object {
69+
"bool": Object {
70+
"must": Array [
71+
Object {
72+
"bool": Object {
73+
"should": Array [
74+
Object {
75+
"term": Object {
76+
"kibana.alert.status": Object {
77+
"value": "active",
78+
},
79+
},
80+
},
81+
Object {
82+
"term": Object {
83+
"kibana.alert.status": Object {
84+
"value": "recovered",
85+
},
86+
},
87+
},
88+
],
89+
},
90+
},
91+
Object {
92+
"bool": Object {
93+
"should": Array [
94+
Object {
95+
"term": Object {
96+
"kibana.alert.rule.uuid": Object {
97+
"value": "test-rule",
98+
},
99+
},
100+
},
101+
],
102+
},
103+
},
104+
],
105+
},
106+
},
107+
"refresh": true,
108+
"script": Object {
109+
"lang": "painless",
110+
"source": "
111+
ctx._source['kibana.alert.flapping'] = false;
112+
ctx._source['kibana.alert.flapping_history'] = new ArrayList();
113+
",
114+
},
115+
},
116+
]
117+
`);
118+
});
119+
120+
test('should throw if either indices or ruleIds is empty', async () => {
121+
await expect(
122+
clearAlertFlappingHistory({
123+
logger,
124+
esClient: clusterClient,
125+
indices: [],
126+
ruleIds: ['test-rule'],
127+
})
128+
).rejects.toThrow('Rule Ids and indices must be provided');
129+
130+
await expect(
131+
clearAlertFlappingHistory({
132+
logger,
133+
esClient: clusterClient,
134+
indices: ['test-index'],
135+
ruleIds: [],
136+
})
137+
).rejects.toThrow('Rule Ids and indices must be provided');
138+
139+
expect(clusterClient.updateByQuery).toHaveBeenCalledTimes(0);
140+
});
141+
142+
test('should throw if could not clear flapping history', async () => {
143+
clusterClient.updateByQuery.mockRejectedValue(Error('something went wrong!'));
144+
await expect(
145+
clearAlertFlappingHistory({
146+
logger,
147+
esClient: clusterClient,
148+
indices: ['test-index'],
149+
ruleIds: ['test-rule'],
150+
})
151+
).rejects.toThrow('something went wrong!');
152+
153+
expect(clusterClient.updateByQuery).toHaveBeenCalledTimes(1);
154+
155+
expect(logger.error).toHaveBeenCalledWith(
156+
'Error clearing alert flapping for indices: test-index, ruleIds: test-rule - something went wrong!'
157+
);
158+
});
159+
160+
test('should retry updateByQuery if it does not finish in 1 attempt', async () => {
161+
clusterClient.updateByQuery.mockResolvedValueOnce({
162+
total: 3,
163+
updated: 1,
164+
});
165+
clusterClient.updateByQuery.mockResolvedValueOnce({
166+
total: 3,
167+
updated: 2,
168+
});
169+
clusterClient.updateByQuery.mockResolvedValueOnce({
170+
total: 3,
171+
updated: 3,
172+
});
173+
174+
await clearAlertFlappingHistory({
175+
logger,
176+
esClient: clusterClient,
177+
indices: ['test-index'],
178+
ruleIds: ['test-rule'],
179+
});
180+
181+
expect(clusterClient.updateByQuery).toHaveBeenCalledTimes(3);
182+
});
183+
});

‎x-pack/platform/plugins/shared/alerting/server/alerts_service/lib/clear_alert_flapping_history.ts‎

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
9+
import type { Logger } from '@kbn/logging';
10+
import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types';
11+
import {
12+
ALERT_RULE_UUID,
13+
ALERT_STATUS,
14+
ALERT_STATUS_ACTIVE,
15+
ALERT_STATUS_RECOVERED,
16+
ALERT_FLAPPING,
17+
ALERT_FLAPPING_HISTORY,
18+
} from '@kbn/rule-data-utils';
19+
20+
export interface ClearAlertFlappingHistoryParams {
21+
indices: string[];
22+
ruleIds: string[];
23+
}
24+
25+
interface ClearAlertFlappingHistoryParamsWithDeps extends ClearAlertFlappingHistoryParams {
26+
logger: Logger;
27+
esClient: ElasticsearchClient;
28+
}
29+
30+
const clearAlertFlappingHistoryQuery = (ruleIds: string[]): QueryDslQueryContainer => {
31+
const shouldStatusTerms = [ALERT_STATUS_ACTIVE, ALERT_STATUS_RECOVERED].map((status) => {
32+
return {
33+
term: {
34+
[ALERT_STATUS]: { value: status },
35+
},
36+
};
37+
});
38+
39+
const shouldRuleIdsTerms = ruleIds.map((ruleId) => {
40+
return {
41+
term: {
42+
[ALERT_RULE_UUID]: { value: ruleId },
43+
},
44+
};
45+
});
46+
47+
return {
48+
bool: {
49+
must: [
50+
{
51+
bool: {
52+
should: shouldStatusTerms,
53+
},
54+
},
55+
{
56+
bool: {
57+
should: shouldRuleIdsTerms,
58+
},
59+
},
60+
],
61+
},
62+
};
63+
};
64+
65+
export const clearAlertFlappingHistory = async (
66+
params: ClearAlertFlappingHistoryParamsWithDeps
67+
) => {
68+
const { indices, ruleIds, esClient, logger } = params;
69+
70+
if (!ruleIds.length || !indices.length) {
71+
throw new Error('Rule Ids and indices must be provided');
72+
}
73+
74+
try {
75+
let total = 0;
76+
77+
for (let retryCount = 0; retryCount < 3; retryCount++) {
78+
const response = await esClient.updateByQuery({
79+
index: indices,
80+
allow_no_indices: true,
81+
conflicts: 'proceed',
82+
script: {
83+
source: `
84+
ctx._source['${ALERT_FLAPPING}'] = false;
85+
ctx._source['${ALERT_FLAPPING_HISTORY}'] = new ArrayList();
86+
`,
87+
lang: 'painless',
88+
},
89+
query: clearAlertFlappingHistoryQuery(ruleIds),
90+
refresh: true,
91+
});
92+
93+
if (total === 0 && response.total === 0) {
94+
logger.debug('No active or recovered alerts matched the query');
95+
break;
96+
}
97+
98+
if (response.total) {
99+
total = response.total;
100+
}
101+
102+
if (response.total === response.updated) {
103+
break;
104+
}
105+
106+
logger.warn(
107+
`Attempt ${retryCount + 1}: Failed to clear flapping ${
108+
(response.total ?? 0) - (response.updated ?? 0)
109+
} of ${response.total}; indices: ${indices}, ruleIds: ${ruleIds}
110+
}`
111+
);
112+
}
113+
114+
if (total === 0) {
115+
return [];
116+
}
117+
} catch (err) {
118+
logger.error(
119+
`Error clearing alert flapping for indices: ${indices}, ruleIds: ${ruleIds} - ${err.message}`
120+
);
121+
throw err;
122+
}
123+
};

0 commit comments

Comments
 (0)