Skip to content

Commit b1a0809

Browse files
rylndrylnd
authored
[9.1] [Detection Engine] Unskip some flaky tests, add better failure messages (elastic#230318) (elastic#231523)
# Backport This will backport the following commits from `main` to `9.1`: - [[Detection Engine] Unskip some flaky tests, add better failure messages (elastic#230318)](elastic#230318) <!--- Backport version: 10.0.1 --> ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sorenlouv/backport) <!--BACKPORT [{"author":{"name":"Ryland Herrick","email":"[email protected]"},"sourceCommit":{"committedDate":"2025-08-11T16:40:18Z","message":"[Detection Engine] Unskip some flaky tests, add better failure messages (elastic#230318)\n\n## Summary\n\nWhile the tests affected by this PR are varied, the changes contained\nhere fall under one of two categories:\n\n1. Unskipping flaky tests\n2. Adding better test assertions (in order to produce more actionable\nfailures later)\n\n### Related Issues\n* Closes elastic#224699\n* Closes elastic#224780\n* Closes elastic#220943\n* Closes elastic#202940\n* Closes elastic#202945\n\n\n### Checklist\n\n- [x] [Unit or functional\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\nwere updated or added to match the most common scenarios","sha":"0a3e7bb41a92b25ca6ae25bef76e53ef634d244b","branchLabelMapping":{"^v9.2.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","backport missing","Team:Detection Engine","backport:version","v9.2.0","v9.1.1","v8.19.1"],"title":"[Detection Engine] Unskip some flaky tests, add better failure messages","number":230318,"url":"https://github.com/elastic/kibana/pull/230318","mergeCommit":{"message":"[Detection Engine] Unskip some flaky tests, add better failure messages (elastic#230318)\n\n## Summary\n\nWhile the tests affected by this PR are varied, the changes contained\nhere fall under one of two categories:\n\n1. Unskipping flaky tests\n2. Adding better test assertions (in order to produce more actionable\nfailures later)\n\n### Related Issues\n* Closes elastic#224699\n* Closes elastic#224780\n* Closes elastic#220943\n* Closes elastic#202940\n* Closes elastic#202945\n\n\n### Checklist\n\n- [x] [Unit or functional\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\nwere updated or added to match the most common scenarios","sha":"0a3e7bb41a92b25ca6ae25bef76e53ef634d244b"}},"sourceBranch":"main","suggestedTargetBranches":["9.1","8.19"],"targetPullRequestStates":[{"branch":"main","label":"v9.2.0","branchLabelMappingKey":"^v9.2.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/230318","number":230318,"mergeCommit":{"message":"[Detection Engine] Unskip some flaky tests, add better failure messages (elastic#230318)\n\n## Summary\n\nWhile the tests affected by this PR are varied, the changes contained\nhere fall under one of two categories:\n\n1. Unskipping flaky tests\n2. Adding better test assertions (in order to produce more actionable\nfailures later)\n\n### Related Issues\n* Closes elastic#224699\n* Closes elastic#224780\n* Closes elastic#220943\n* Closes elastic#202940\n* Closes elastic#202945\n\n\n### Checklist\n\n- [x] [Unit or functional\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\nwere updated or added to match the most common scenarios","sha":"0a3e7bb41a92b25ca6ae25bef76e53ef634d244b"}},{"branch":"9.1","label":"v9.1.1","branchLabelMappingKey":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"8.19","label":"v8.19.1","branchLabelMappingKey":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"}]}] BACKPORT-->
1 parent 2e4ac93 commit b1a0809

File tree

9 files changed

+224
-229
lines changed

9 files changed

+224
-229
lines changed

‎x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/workflows/basic_license_essentials_tier/rule_exceptions_execution.ts‎

Lines changed: 13 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,7 @@ export default ({ getService }: FtrProviderContext) => {
122122
await waitForRuleSuccess({ supertest, log, id: createdId });
123123
await waitForAlertsToBePresent(supertest, log, 10, [createdId]);
124124
const alertsOpen = await getAlertsByIds(supertest, log, [createdId]);
125-
expect(alertsOpen.hits.hits.length).toEqual(10);
125+
expect(alertsOpen.hits.hits).toHaveLength(10);
126126
});
127127

128128
it('should be able to execute against an exception list that does include valid entries and get back 0 alerts', async () => {
@@ -149,7 +149,7 @@ export default ({ getService }: FtrProviderContext) => {
149149
],
150150
]);
151151
const alertsOpen = await getOpenAlerts(supertest, log, es, createdRule);
152-
expect(alertsOpen.hits.hits.length).toEqual(0);
152+
expect(alertsOpen.hits.hits).toHaveLength(0);
153153
});
154154

155155
it('should be able to execute against an exception list that does include valid case sensitive entries and get back 0 alerts', async () => {
@@ -201,10 +201,10 @@ export default ({ getService }: FtrProviderContext) => {
201201
const alertsOpen2 = await getOpenAlerts(supertest, log, es, createdRule2);
202202
// Expect alerts here because all values are "Ubuntu"
203203
// and exception is one of ["ubuntu"]
204-
expect(alertsOpen.hits.hits.length).toEqual(10);
204+
expect(alertsOpen.hits.hits).toHaveLength(10);
205205
// Expect no alerts here because all values are "Ubuntu"
206206
// and exception is one of ["ubuntu", "Ubuntu"]
207-
expect(alertsOpen2.hits.hits.length).toEqual(0);
207+
expect(alertsOpen2.hits.hits).toHaveLength(0);
208208
});
209209

210210
it('generates no alerts when an exception is added for an EQL rule', async () => {
@@ -223,7 +223,7 @@ export default ({ getService }: FtrProviderContext) => {
223223
],
224224
]);
225225
const alertsOpen = await getOpenAlerts(supertest, log, es, createdRule);
226-
expect(alertsOpen.hits.hits.length).toEqual(0);
226+
expect(alertsOpen.hits.hits).toHaveLength(0);
227227
});
228228

229229
it('generates no alerts when an exception is added for a threshold rule', async () => {
@@ -245,7 +245,7 @@ export default ({ getService }: FtrProviderContext) => {
245245
],
246246
]);
247247
const alertsOpen = await getOpenAlerts(supertest, log, es, createdRule);
248-
expect(alertsOpen.hits.hits.length).toEqual(0);
248+
expect(alertsOpen.hits.hits).toHaveLength(0);
249249
});
250250

251251
it('generates no alerts when an exception is added for a threat match rule', async () => {
@@ -288,8 +288,9 @@ export default ({ getService }: FtrProviderContext) => {
288288
],
289289
]);
290290
const alertsOpen = await getOpenAlerts(supertest, log, es, createdRule);
291-
expect(alertsOpen.hits.hits.length).toEqual(0);
291+
expect(alertsOpen.hits.hits).toHaveLength(0);
292292
});
293+
293294
describe('rules with value list exceptions', () => {
294295
beforeEach(async () => {
295296
await createListsIndex(supertest, log);
@@ -328,7 +329,7 @@ export default ({ getService }: FtrProviderContext) => {
328329
],
329330
]);
330331
const alertsOpen = await getOpenAlerts(supertest, log, es, createdRule);
331-
expect(alertsOpen.hits.hits.length).toEqual(0);
332+
expect(alertsOpen.hits.hits).toHaveLength(0);
332333
});
333334

334335
it('generates no alerts when a value list exception is added for a threat match rule', async () => {
@@ -376,7 +377,7 @@ export default ({ getService }: FtrProviderContext) => {
376377
],
377378
]);
378379
const alertsOpen = await getOpenAlerts(supertest, log, es, createdRule);
379-
expect(alertsOpen.hits.hits.length).toEqual(0);
380+
expect(alertsOpen.hits.hits).toHaveLength(0);
380381
});
381382

382383
it('generates no alerts when a value list exception is added for a threshold rule', async () => {
@@ -413,7 +414,7 @@ export default ({ getService }: FtrProviderContext) => {
413414
],
414415
]);
415416
const alertsOpen = await getOpenAlerts(supertest, log, es, createdRule);
416-
expect(alertsOpen.hits.hits.length).toEqual(0);
417+
expect(alertsOpen.hits.hits).toHaveLength(0);
417418
});
418419

419420
it('generates no alerts when a value list exception is added for an EQL rule', async () => {
@@ -438,8 +439,9 @@ export default ({ getService }: FtrProviderContext) => {
438439
],
439440
]);
440441
const alertsOpen = await getOpenAlerts(supertest, log, es, createdRule);
441-
expect(alertsOpen.hits.hits.length).toEqual(0);
442+
expect(alertsOpen.hits.hits).toHaveLength(0);
442443
});
444+
443445
it('should Not allow deleting value list when there are references and ignoreReferences is false', async () => {
444446
const valueListId = 'value-list-id.txt';
445447
await importFile(supertest, log, 'keyword', ['suricata-sensor-amsterdam'], valueListId);

‎x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql.ts‎

Lines changed: 29 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -82,8 +82,7 @@ export default ({ getService }: FtrProviderContext) => {
8282
const auditPath = dataPathBuilder.getPath('auditbeat/hosts');
8383
const packetBeatPath = dataPathBuilder.getPath('packetbeat/default');
8484

85-
// FLAKY: https://github.com/elastic/kibana/issues/220943
86-
describe.skip('@ess @serverless @serverlessQA EQL type rules', () => {
85+
describe('@ess @serverless @serverlessQA EQL type rules', () => {
8786
const { indexListOfDocuments } = dataGeneratorFactory({
8887
es,
8988
index: 'ecs_compliant',
@@ -120,7 +119,7 @@ export default ({ getService }: FtrProviderContext) => {
120119
};
121120
const createdRule = await createRule(supertest, log, rule);
122121
const alerts = await getAlerts(supertest, log, es, createdRule);
123-
kbnExpect(alerts.hits.hits.length).eql(1);
122+
expect(alerts.hits.hits).toHaveLength(1);
124123
const fullAlert = alerts.hits.hits[0]._source;
125124
if (!fullAlert) {
126125
return kbnExpect(fullAlert).to.be.ok();
@@ -289,7 +288,7 @@ export default ({ getService }: FtrProviderContext) => {
289288
};
290289
const { previewId } = await previewRule({ supertest, rule });
291290
const previewAlerts = await getPreviewAlerts({ es, previewId, size: maxAlerts * 2 });
292-
kbnExpect(previewAlerts.length).eql(maxAlerts);
291+
expect(previewAlerts).toHaveLength(maxAlerts);
293292
});
294293

295294
it('generates max alerts warning when circuit breaker is hit', async () => {
@@ -308,7 +307,7 @@ export default ({ getService }: FtrProviderContext) => {
308307
};
309308
const { previewId } = await previewRule({ supertest, rule });
310309
const previewAlerts = await getPreviewAlerts({ es, previewId });
311-
kbnExpect(previewAlerts.length).eql(1);
310+
expect(previewAlerts).toHaveLength(1);
312311
const fullAlert = previewAlerts[0]._source;
313312
if (!fullAlert) {
314313
return kbnExpect(fullAlert).to.be.ok();
@@ -378,7 +377,7 @@ export default ({ getService }: FtrProviderContext) => {
378377
};
379378
const { previewId } = await previewRule({ supertest, rule });
380379
const previewAlerts = await getPreviewAlerts({ es, previewId });
381-
kbnExpect(previewAlerts.length).eql(3);
380+
expect(previewAlerts).toHaveLength(3);
382381

383382
const createdAtHits = previewAlerts.map((hit) => hit._source?.created_at).sort();
384383
kbnExpect(createdAtHits).to.eql([1622676785, 1622676790, 1622676795]);
@@ -392,7 +391,7 @@ export default ({ getService }: FtrProviderContext) => {
392391
};
393392
const { previewId } = await previewRule({ supertest, rule });
394393
const previewAlerts = await getPreviewAlerts({ es, previewId });
395-
kbnExpect(previewAlerts.length).eql(3);
394+
expect(previewAlerts).toHaveLength(3);
396395

397396
const createdAtHits = previewAlerts.map((hit) => hit._source?.locale);
398397
kbnExpect(createdAtHits).to.eql(['es', 'pt', 'ua']);
@@ -672,7 +671,7 @@ export default ({ getService }: FtrProviderContext) => {
672671

673672
const previewAlerts = await getPreviewAlerts({ es, previewId, sort: ['agent.name'] });
674673

675-
kbnExpect(previewAlerts).to.have.length(3);
674+
expect(previewAlerts).toHaveLength(3);
676675

677676
const buildingBlockAlerts = previewAlerts.filter(
678677
(alert) => alert._source?.['kibana.alert.building_block_type']
@@ -716,11 +715,11 @@ export default ({ getService }: FtrProviderContext) => {
716715
// For EQL rules, max_alerts is the maximum number of detected sequences: each sequence has a building block
717716
// alert for each event in the sequence, so max_alerts=200 results in 400 building blocks in addition to
718717
// 200 regular alerts
719-
kbnExpect(previewAlerts.length).eql(maxAlerts * 3);
718+
expect(previewAlerts).toHaveLength(maxAlerts * 3);
720719
const shellAlerts = previewAlerts.filter((alert) => alert._source?.[ALERT_DEPTH] === 2);
721720
const buildingBlocks = previewAlerts.filter((alert) => alert._source?.[ALERT_DEPTH] === 1);
722-
kbnExpect(shellAlerts.length).eql(maxAlerts);
723-
kbnExpect(buildingBlocks.length).eql(maxAlerts * 2);
721+
expect(shellAlerts).toHaveLength(maxAlerts);
722+
expect(buildingBlocks).toHaveLength(maxAlerts * 2);
724723
});
725724

726725
it('generates alerts when an index name contains special characters to encode', async () => {
@@ -730,7 +729,7 @@ export default ({ getService }: FtrProviderContext) => {
730729
};
731730
const { previewId } = await previewRule({ supertest, rule });
732731
const previewAlerts = await getPreviewAlerts({ es, previewId });
733-
kbnExpect(previewAlerts.length).eql(1);
732+
expect(previewAlerts).toHaveLength(1);
734733
});
735734

736735
it('uses the provided filters', async () => {
@@ -776,7 +775,7 @@ export default ({ getService }: FtrProviderContext) => {
776775
};
777776
const { previewId } = await previewRule({ supertest, rule });
778777
const previewAlerts = await getPreviewAlerts({ es, previewId });
779-
kbnExpect(previewAlerts.length).eql(2);
778+
expect(previewAlerts).toHaveLength(2);
780779
});
781780

782781
describe('with host risk index', () => {
@@ -795,7 +794,7 @@ export default ({ getService }: FtrProviderContext) => {
795794
};
796795
const { previewId } = await previewRule({ supertest, rule });
797796
const previewAlerts = await getPreviewAlerts({ es, previewId });
798-
kbnExpect(previewAlerts.length).eql(1);
797+
expect(previewAlerts).toHaveLength(1);
799798
const fullAlert = previewAlerts[0]._source;
800799
if (!fullAlert) {
801800
return kbnExpect(fullAlert).to.be.ok();
@@ -850,7 +849,7 @@ export default ({ getService }: FtrProviderContext) => {
850849
kbnExpect(_log.warnings).to.eql([expectedWarning]);
851850

852851
const previewAlerts = await getPreviewAlerts({ es, previewId });
853-
kbnExpect(previewAlerts.length).to.be.greaterThan(0);
852+
expect(previewAlerts).not.toHaveLength(0);
854853
});
855854

856855
it('specifying only timestamp_override results in alert creation with an kbnExpect.expected warning', async () => {
@@ -868,7 +867,7 @@ export default ({ getService }: FtrProviderContext) => {
868867
kbnExpect(_log.warnings).to.eql([expectedWarning]);
869868

870869
const previewAlerts = await getPreviewAlerts({ es, previewId });
871-
kbnExpect(previewAlerts.length).to.be.greaterThan(0);
870+
expect(previewAlerts).not.toHaveLength(0);
872871
});
873872

874873
it('specifying both timestamp_override and timestamp_field results in alert creation with an kbnExpect.expected warning', async () => {
@@ -887,7 +886,7 @@ export default ({ getService }: FtrProviderContext) => {
887886
kbnExpect(_log.warnings).to.eql([expectedWarning]);
888887

889888
const previewAlerts = await getPreviewAlerts({ es, previewId });
890-
kbnExpect(previewAlerts.length).to.be.greaterThan(0);
889+
expect(previewAlerts).not.toHaveLength(0);
891890
});
892891
});
893892

@@ -959,7 +958,7 @@ export default ({ getService }: FtrProviderContext) => {
959958
kbnExpect(_log.warnings).to.be.empty();
960959
const previewAlerts = await getPreviewAlerts({ es, previewId });
961960

962-
kbnExpect(previewAlerts).to.have.length(3);
961+
expect(previewAlerts).toHaveLength(3);
963962
});
964963
});
965964

@@ -1034,7 +1033,7 @@ export default ({ getService }: FtrProviderContext) => {
10341033

10351034
const createdRule = await createRule(supertest, log, rule);
10361035
const alerts = await getAlerts(supertest, log, es, createdRule);
1037-
kbnExpect(alerts.hits.hits.length).equal(3);
1036+
expect(alerts.hits.hits).toHaveLength(3);
10381037
kbnExpect(alerts.hits.hits[0]?._source?.[ALERT_RULE_EXECUTION_TYPE]).equal('scheduled');
10391038

10401039
const backfill = await scheduleRuleRun(supertest, [createdRule.id], {
@@ -1044,7 +1043,7 @@ export default ({ getService }: FtrProviderContext) => {
10441043

10451044
await waitForBackfillExecuted(backfill, [createdRule.id], { supertest, log });
10461045
const allNewAlerts = await getAlerts(supertest, log, es, createdRule);
1047-
kbnExpect(allNewAlerts.hits.hits.length).equal(6);
1046+
expect(allNewAlerts.hits.hits).toHaveLength(6);
10481047
kbnExpect(allNewAlerts.hits.hits[5]?._source?.[ALERT_RULE_EXECUTION_TYPE]).equal('manual');
10491048

10501049
const secondBackfill = await scheduleRuleRun(supertest, [createdRule.id], {
@@ -1054,7 +1053,7 @@ export default ({ getService }: FtrProviderContext) => {
10541053

10551054
await waitForBackfillExecuted(secondBackfill, [createdRule.id], { supertest, log });
10561055
const allNewAlertsAfter2ManualRuns = await getAlerts(supertest, log, es, createdRule);
1057-
kbnExpect(allNewAlertsAfter2ManualRuns.hits.hits.length).equal(6);
1056+
expect(allNewAlertsAfter2ManualRuns.hits.hits).toHaveLength(6);
10581057
});
10591058

10601059
it('does not alert if the manual run overlaps with a previous scheduled rule execution', async () => {
@@ -1093,7 +1092,7 @@ export default ({ getService }: FtrProviderContext) => {
10931092
const createdRule = await createRule(supertest, log, rule);
10941093
const alerts = await getAlerts(supertest, log, es, createdRule);
10951094

1096-
kbnExpect(alerts.hits.hits.length).equal(3);
1095+
expect(alerts.hits.hits).toHaveLength(3);
10971096

10981097
const backfill = await scheduleRuleRun(supertest, [createdRule.id], {
10991098
startDate: moment(firstTimestamp).subtract(5, 'm'),
@@ -1102,7 +1101,7 @@ export default ({ getService }: FtrProviderContext) => {
11021101

11031102
await waitForBackfillExecuted(backfill, [createdRule.id], { supertest, log });
11041103
const allNewAlerts = await getAlerts(supertest, log, es, createdRule);
1105-
kbnExpect(allNewAlerts.hits.hits.length).equal(3);
1104+
expect(allNewAlerts.hits.hits).toHaveLength(3);
11061105
});
11071106

11081107
it('supression per rule execution should work for manual rule runs', async () => {
@@ -1146,7 +1145,7 @@ export default ({ getService }: FtrProviderContext) => {
11461145
const createdRule = await createRule(supertest, log, rule);
11471146
const alerts = await getAlerts(supertest, log, es, createdRule);
11481147

1149-
kbnExpect(alerts.hits.hits.length).equal(0);
1148+
expect(alerts.hits.hits).toHaveLength(0);
11501149

11511150
const backfill = await scheduleRuleRun(supertest, [createdRule.id], {
11521151
startDate: moment(firstTimestamp).subtract(5, 'm'),
@@ -1155,7 +1154,7 @@ export default ({ getService }: FtrProviderContext) => {
11551154

11561155
await waitForBackfillExecuted(backfill, [createdRule.id], { supertest, log });
11571156
const allNewAlerts = await getAlerts(supertest, log, es, createdRule);
1158-
kbnExpect(allNewAlerts.hits.hits.length).equal(1);
1157+
expect(allNewAlerts.hits.hits).toHaveLength(1);
11591158

11601159
kbnExpect(allNewAlerts.hits.hits[0]._source?.[ALERT_SUPPRESSION_DOCS_COUNT]).equal(2);
11611160
});
@@ -1191,7 +1190,7 @@ export default ({ getService }: FtrProviderContext) => {
11911190
const createdRule = await createRule(supertest, log, rule);
11921191
const alerts = await getAlerts(supertest, log, es, createdRule);
11931192

1194-
kbnExpect(alerts.hits.hits.length).equal(0);
1193+
expect(alerts.hits.hits).toHaveLength(0);
11951194

11961195
// generate alert in the past
11971196
const backfill = await scheduleRuleRun(supertest, [createdRule.id], {
@@ -1200,7 +1199,7 @@ export default ({ getService }: FtrProviderContext) => {
12001199
});
12011200
await waitForBackfillExecuted(backfill, [createdRule.id], { supertest, log });
12021201
const allNewAlerts = await getAlerts(supertest, log, es, createdRule);
1203-
kbnExpect(allNewAlerts.hits.hits.length).equal(1);
1202+
expect(allNewAlerts.hits.hits).toHaveLength(1);
12041203

12051204
// now we will ingest new event, and manual rule run should update original alert
12061205
const secondDocument = {
@@ -1220,9 +1219,9 @@ export default ({ getService }: FtrProviderContext) => {
12201219

12211220
await waitForBackfillExecuted(secondBackfill, [createdRule.id], { supertest, log });
12221221
const updatedAlerts = await getAlerts(supertest, log, es, createdRule);
1223-
kbnExpect(updatedAlerts.hits.hits.length).equal(1);
1222+
expect(updatedAlerts.hits.hits).toHaveLength(1);
12241223

1225-
kbnExpect(updatedAlerts.hits.hits.length).equal(1);
1224+
expect(updatedAlerts.hits.hits).toHaveLength(1);
12261225

12271226
kbnExpect(updatedAlerts.hits.hits[0]._source?.[ALERT_SUPPRESSION_DOCS_COUNT]).equal(1);
12281227
});
@@ -1247,7 +1246,7 @@ export default ({ getService }: FtrProviderContext) => {
12471246

12481247
const requests = logs[0].requests;
12491248

1250-
kbnExpect(requests).to.have.length(1);
1249+
expect(requests).toHaveLength(1);
12511250
kbnExpect(requests![0].description).to.be('EQL request to find all matches');
12521251
kbnExpect(requests![0].request).to.contain(
12531252
'POST /auditbeat-*/_eql/search?allow_no_indices=true'

0 commit comments

Comments
 (0)