Skip to content

Commit 02e6493

Browse files
kibanamachineCAWilson94
kibanamachine
and
CAWilson94
authored
[9.2] [Privmon] Deletion Detection and Monitoring Sources FTR Test Updates (#240051) (#240853)
# Backport This will backport the following commits from `main` to `9.2`: - [[Privmon] Deletion Detection and Monitoring Sources FTR Test Updates (#240051)](#240051) <!--- Backport version: 9.6.6 --> ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sorenlouv/backport) <!--BACKPORT [{"author":{"name":"Charlotte Alexandra Wilson","email":"[email protected]"},"sourceCommit":{"committedDate":"2025-10-27T15:39:27Z","message":"[Privmon] Deletion Detection and Monitoring Sources FTR Test Updates (#240051)\n\nThis PR introduces FTR Testing for deletion detection in the privileged\nuser monitoring, integrations sync, feature.\n\n### Key Updates: \n\n- Coverage to test when a user is deleted, they are correctly handled\nduring a full sync window - labels updated, privileged status set to\nfalse. This also tests the full sync detection functionality.\n- Coverage to test the case of start + complete window outside of the\ncurrent batch of users, soft deleting ALL users.\n- Updated utilities functions with some common helpers and constants to\ncreate the full sync window, delete integrations user and reduce hard\ncoded values.\n- Updated default entity source tests to include AD.\n\n### 🐞 Bugs Found & Fixed During Test Creation\n\n**First-run handling:**\n- Previously, the fallback logic inside “detect full sync” didn’t handle\nthe first run correctly.\n- When the SO had no previous sync, it treated `fullSync ===\nlastCompletedEvent`\n- and skipped the first full-sync run.\n- This has been fixed to ensure the initial run executes properly.\n\n**The fix - Last full sync source**\n* Updated lastFullSync to read only from the Saved Object (SO).\n* The checks for a completed marker now live inside the deletion\ndetection logic itself.","sha":"41a5600a8278e28aab5e14084814a354b62ead39","branchLabelMapping":{"^v9.3.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","Team:Entity Analytics","backport:version","v9.2.0","v9.3.0"],"title":"[Privmon] Deletion Detection and Monitoring Sources FTR Test Updates","number":240051,"url":"https://github.com/elastic/kibana/pull/240051","mergeCommit":{"message":"[Privmon] Deletion Detection and Monitoring Sources FTR Test Updates (#240051)\n\nThis PR introduces FTR Testing for deletion detection in the privileged\nuser monitoring, integrations sync, feature.\n\n### Key Updates: \n\n- Coverage to test when a user is deleted, they are correctly handled\nduring a full sync window - labels updated, privileged status set to\nfalse. This also tests the full sync detection functionality.\n- Coverage to test the case of start + complete window outside of the\ncurrent batch of users, soft deleting ALL users.\n- Updated utilities functions with some common helpers and constants to\ncreate the full sync window, delete integrations user and reduce hard\ncoded values.\n- Updated default entity source tests to include AD.\n\n### 🐞 Bugs Found & Fixed During Test Creation\n\n**First-run handling:**\n- Previously, the fallback logic inside “detect full sync” didn’t handle\nthe first run correctly.\n- When the SO had no previous sync, it treated `fullSync ===\nlastCompletedEvent`\n- and skipped the first full-sync run.\n- This has been fixed to ensure the initial run executes properly.\n\n**The fix - Last full sync source**\n* Updated lastFullSync to read only from the Saved Object (SO).\n* The checks for a completed marker now live inside the deletion\ndetection logic itself.","sha":"41a5600a8278e28aab5e14084814a354b62ead39"}},"sourceBranch":"main","suggestedTargetBranches":["9.2"],"targetPullRequestStates":[{"branch":"9.2","label":"v9.2.0","branchLabelMappingKey":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"main","label":"v9.3.0","branchLabelMappingKey":"^v9.3.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/240051","number":240051,"mergeCommit":{"message":"[Privmon] Deletion Detection and Monitoring Sources FTR Test Updates (#240051)\n\nThis PR introduces FTR Testing for deletion detection in the privileged\nuser monitoring, integrations sync, feature.\n\n### Key Updates: \n\n- Coverage to test when a user is deleted, they are correctly handled\nduring a full sync window - labels updated, privileged status set to\nfalse. This also tests the full sync detection functionality.\n- Coverage to test the case of start + complete window outside of the\ncurrent batch of users, soft deleting ALL users.\n- Updated utilities functions with some common helpers and constants to\ncreate the full sync window, delete integrations user and reduce hard\ncoded values.\n- Updated default entity source tests to include AD.\n\n### 🐞 Bugs Found & Fixed During Test Creation\n\n**First-run handling:**\n- Previously, the fallback logic inside “detect full sync” didn’t handle\nthe first run correctly.\n- When the SO had no previous sync, it treated `fullSync ===\nlastCompletedEvent`\n- and skipped the first full-sync run.\n- This has been fixed to ensure the initial run executes properly.\n\n**The fix - Last full sync source**\n* Updated lastFullSync to read only from the Saved Object (SO).\n* The checks for a completed marker now live inside the deletion\ndetection logic itself.","sha":"41a5600a8278e28aab5e14084814a354b62ead39"}}]}] BACKPORT--> Co-authored-by: Charlotte Alexandra Wilson <[email protected]>
1 parent 99b0758 commit 02e6493

File tree

3 files changed

+242
-53
lines changed
  • x-pack/solutions/security
    • plugins/security_solution/server/lib/entity_analytics/privilege_monitoring/data_sources/sync/integrations/sync_markers
    • test/security_solution_api_integration/test_suites/entity_analytics/monitoring/trial_license_complete_tier

3 files changed

+242
-53
lines changed

‎x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/privilege_monitoring/data_sources/sync/integrations/sync_markers/sync_markers.ts‎

Lines changed: 1 addition & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -58,17 +58,9 @@ export const createSyncMarkersService = (
5858
const getLastFullSyncMarker = async (
5959
source: MonitoringEntitySource
6060
): Promise<string | undefined> => {
61-
// Try from the SO first
6261
const fromSO = await monitoringIndexSourceClient.getLastFullSyncMarker(source);
6362
if (fromSO) return fromSO;
64-
65-
// Fallback: search index
66-
const fromIndex = await findLastEventMarkerInIndex(source, 'completed');
67-
if (!fromIndex) return undefined;
68-
69-
// Update the SO for next time
70-
await updateLastFullSyncMarker(source, fromIndex);
71-
return fromIndex;
63+
return undefined;
7264
};
7365

7466
const findLastEventMarkerInIndex = async (
@@ -100,7 +92,6 @@ export const createSyncMarkersService = (
10092
const shouldUpdate = !lastFullSync || latestCompletedEvent > lastFullSync;
10193

10294
if (!shouldUpdate) return false;
103-
10495
await updateLastFullSyncMarker(source, latestCompletedEvent);
10596
return true;
10697
};

‎x-pack/solutions/security/test/security_solution_api_integration/test_suites/entity_analytics/monitoring/trial_license_complete_tier/engine.ts‎

Lines changed: 66 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -438,7 +438,7 @@ export default ({ getService }: FtrProviderContext) => {
438438
);
439439
// Set timestamps to be within last month so they are included in sync (default first run is now - 1M)
440440
await privMonUtils.integrationsSync.updateIntegrationsUsersWithRelativeTimestamps({
441-
indexPattern: 'logs-entityanalytics_okta.user-default',
441+
indexPattern: privMonUtils.integrationsSync.OKTA_INDEX,
442442
});
443443
await enablePrivmonSetting(kibanaServer);
444444
await privMonUtils.initPrivMonEngine();
@@ -462,8 +462,7 @@ export default ({ getService }: FtrProviderContext) => {
462462
expect(monitoringSource?.name).toBe(
463463
'.entity_analytics.monitoring.sources.entityanalytics_okta-default'
464464
);
465-
await privMonUtils.scheduleMonitoringEngineNow({ ignoreConflict: true });
466-
await privMonUtils.waitForSyncTaskRun();
465+
await privMonUtils.runSync();
467466

468467
const { body: usersBefore } = await api.listPrivMonUsers({ query: {} });
469468
// each user should be privileged and have correct source
@@ -500,13 +499,12 @@ export default ({ getService }: FtrProviderContext) => {
500499

501500
// update okta user to non-privileged, to test sync updates
502501
await privMonUtils.integrationsSync.setIntegrationUserPrivilege({
503-
id: 'AZlHQD20hY07UD0HNBs-',
502+
id: privMonUtils.integrationsSync.OKTA_USER_IDS.mable,
504503
isPrivileged: false,
505-
indexPattern: 'logs-entityanalytics_okta.user-default',
504+
indexPattern: privMonUtils.integrationsSync.OKTA_INDEX,
506505
});
507506
// schedule another sync
508-
await privMonUtils.scheduleMonitoringEngineNow({ ignoreConflict: true });
509-
await privMonUtils.waitForSyncTaskRun();
507+
await privMonUtils.runSync();
510508
const { body: usersAfter } = await api.listPrivMonUsers({ query: {} });
511509

512510
// find the updated user
@@ -534,24 +532,25 @@ export default ({ getService }: FtrProviderContext) => {
534532
});
535533

536534
it('should update and create users within lastProcessedMarker range during update detection ', async () => {
537-
const oktaIndex = 'logs-entityanalytics_okta.user-default';
538-
const IDS = {
539-
devon: 'AZmLBcGV9XhZAwOqZV5t', // Devan.Nienow
540-
elinor: 'AZmLBcGV9XhZAwOqZV5u', // Elinor.Johnston-Shanahan
541-
kaelyn: 'AZmLBcGV9XhZAwOqZV5s', // Kaelyn.Shanahan
542-
bennett: 'AZmLBcGV9XhZAwOqZV5y', // Bennett.Becker
543-
};
544535
// --- Timestamps
545-
const nowMinus1M1D = await privMonUtils.integrationsSync.dateOffsetFromNow({
536+
const nowMinus1M1D = await privMonUtils.integrationsSync.dateOffsetFrom({
546537
months: 2,
547538
days: 1,
548539
});
549-
const nowMinus2M = await privMonUtils.integrationsSync.dateOffsetFromNow({ months: 2 });
550-
const nowMinus1w = await privMonUtils.integrationsSync.dateOffsetFromNow({ days: 7 });
551-
const nowMinus6d = await privMonUtils.integrationsSync.dateOffsetFromNow({ days: 6 });
540+
const nowMinus2M = await privMonUtils.integrationsSync.dateOffsetFrom({ months: 2 });
541+
const nowMinus1w = await privMonUtils.integrationsSync.dateOffsetFrom({ days: 7 });
542+
const nowMinus6d = await privMonUtils.integrationsSync.dateOffsetFrom({ days: 6 });
552543
// PHASE 1: Push two users out of range, sync => expect 4 privileged remain
553-
await privMonUtils.integrationsSync.setTimestamp(IDS.devon, nowMinus1M1D, oktaIndex);
554-
await privMonUtils.integrationsSync.setTimestamp(IDS.elinor, nowMinus2M, oktaIndex);
544+
await privMonUtils.integrationsSync.setTimestamp(
545+
privMonUtils.integrationsSync.OKTA_USER_IDS.devon,
546+
nowMinus1M1D,
547+
privMonUtils.integrationsSync.OKTA_INDEX
548+
);
549+
await privMonUtils.integrationsSync.setTimestamp(
550+
privMonUtils.integrationsSync.OKTA_USER_IDS.elinor,
551+
nowMinus2M,
552+
privMonUtils.integrationsSync.OKTA_INDEX
553+
);
555554
await privMonUtils.runSync();
556555
const snapA = await privMonUtils.integrationsSync.expectUserCount(4);
557556

@@ -561,39 +560,73 @@ export default ({ getService }: FtrProviderContext) => {
561560
expect(new Set(snapB)).toEqual(new Set(snapA));
562561

563562
const markerAfterPhase2 = await privMonUtils.integrationsSync.getLastProcessedMarker(
564-
oktaIndex
563+
privMonUtils.integrationsSync.OKTA_INDEX
565564
);
566565
expect(markerAfterPhase2).toBe(
567566
privMonUtils.integrationsSync.DEFAULT_INTEGRATIONS_RELATIVE_TIMESTAMP
568567
);
569568

570569
// PHASE 3: Bring one user back in-range, sync => last processed marker updates to that timestamp
571-
await privMonUtils.integrationsSync.setTimestamp(IDS.kaelyn, nowMinus1w, oktaIndex);
570+
await privMonUtils.integrationsSync.setTimestamp(
571+
privMonUtils.integrationsSync.OKTA_USER_IDS.kaelyn,
572+
nowMinus1w,
573+
privMonUtils.integrationsSync.OKTA_INDEX
574+
);
572575
await privMonUtils.runSync();
573576
const markerAfterPhase3 = await privMonUtils.integrationsSync.getLastProcessedMarker(
574-
oktaIndex
577+
privMonUtils.integrationsSync.OKTA_INDEX
575578
);
576579
expect(markerAfterPhase3).toBe(nowMinus1w);
577580

578581
// PHASE 4: Flip a non-privileged user to privileged + in-range, sync => count 5, last processed marker updates
579582
await privMonUtils.integrationsSync.setIntegrationUserPrivilege({
580-
id: IDS.bennett,
583+
id: privMonUtils.integrationsSync.OKTA_USER_IDS.bennett,
581584
isPrivileged: true,
582-
indexPattern: 'logs-entityanalytics_okta.user-default',
585+
indexPattern: privMonUtils.integrationsSync.OKTA_INDEX,
583586
});
584-
await privMonUtils.integrationsSync.setTimestamp(IDS.bennett, nowMinus6d, oktaIndex);
587+
await privMonUtils.integrationsSync.setTimestamp(
588+
privMonUtils.integrationsSync.OKTA_USER_IDS.bennett,
589+
nowMinus6d,
590+
privMonUtils.integrationsSync.OKTA_INDEX
591+
);
585592

586593
await privMonUtils.runSync();
587594
await privMonUtils.integrationsSync.expectUserCount(5);
588595

589596
const markerAfterPhase4 = await privMonUtils.integrationsSync.getLastProcessedMarker(
590-
oktaIndex
597+
privMonUtils.integrationsSync.OKTA_INDEX
591598
);
592599
expect(markerAfterPhase4).toBe(nowMinus6d);
593600
});
594601

595-
it.skip('deletion detection should delete users on full sync', async () => {
596-
// placeholder for deletion detection
602+
it('deletes missing users during a full sync window', async () => {
603+
const EXPECTED_DELETED_USERNAME = 'Mable.Mann';
604+
// Initial sync to establish users - deletion detection NOT expected
605+
await privMonUtils.runSync();
606+
// Create sync window
607+
await privMonUtils.integrationsSync.createFullSyncWindowFromOffsets();
608+
// Remove on user from source, simulating deletion
609+
await privMonUtils.integrationsSync.deleteIntegrationUser({
610+
id: privMonUtils.integrationsSync.OKTA_USER_IDS.mable,
611+
indexPattern: privMonUtils.integrationsSync.OKTA_INDEX,
612+
});
613+
// Run sync - deletion detection expected to run and remove the user
614+
await privMonUtils.runSync();
615+
const users = await privMonUtils.integrationsSync.expectUserCount(6);
616+
const mable = privMonUtils.findUser(users, EXPECTED_DELETED_USERNAME);
617+
expect(mable).toBeDefined();
618+
privMonUtils.assertIsPrivileged(mable, false);
619+
expect(mable?.entity_analytics_monitoring?.labels).toEqual([]);
620+
await privMonUtils.integrationsSync.createFullSyncWindowFromOffsets({
621+
startOffsetMinutes: -40,
622+
completeOffsetMinutes: -45,
623+
});
624+
await privMonUtils.runSync();
625+
const usersAfter = await privMonUtils.integrationsSync.expectUserCount(6);
626+
usersAfter.forEach((u: any) => {
627+
expect(u.user.is_privileged).toBe(false);
628+
});
629+
await privMonUtils.integrationsSync.cleanupEventsIndex();
597630
});
598631
});
599632

@@ -614,14 +647,15 @@ export default ({ getService }: FtrProviderContext) => {
614647
expect(names).toEqual(
615648
expect.arrayContaining([
616649
'.entity_analytics.monitoring.sources.entityanalytics_okta-default',
617-
// '.entity_analytics.monitoring.sources.ad-default',
650+
'.entity_analytics.monitoring.sources.entityanalytics_ad-default',
618651
'.entity_analytics.monitoring.users-default',
619652
])
620653
);
621654
expect(syncMarkersIndices).toEqual(
622655
expect.arrayContaining([
623-
'logs-entityanalytics_okta.entity-default',
624-
// '.entity_analytics.monitoring.sources.ad-default',
656+
undefined, // default users source has no sync marker since index does not use sync markers
657+
'logs-entityanalytics_okta.entity-default', // okta sync markers source
658+
'logs-entityanalytics_ad.user-default', // ad sync markers source
625659
])
626660
);
627661
});

0 commit comments

Comments
 (0)