[9.1] [Privmon] Add Functional Tests for Index Sync and init API Access (elastic#229137) (elastic#231179)

# Backport

This will backport the following commits from `main` to `9.1`:
- [[Privmon] Add Functional Tests for Index Sync and init API Access
(elastic#229137)](elastic#229137)

<!--- Backport version: 9.6.6 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sorenlouv/backport)

<!--BACKPORT [{"author":{"name":"Charlotte Alexandra
Wilson","email":"[email protected]"},"sourceCommit":{"committedDate":"2025-08-08T15:43:04Z","message":"[Privmon]
Add Functional Tests for Index Sync and init API Access
(elastic#229137)\n\nRebased with this
PR:\nhttps://github.com/elastic/pull/229019/commits\n\n##
Summary\nThis PR introduces some basic FTR tests for the privilege
monitor\nsyncing.\n1. ✅ Plain Index Sync Test - Verifies that the
Privileged Monitoring\nsystem can correctly:\n\t•\tSync user data from a
plain (non-integration) index,\n\t•\tDeduplicate users,\n\t•\tPersist
the expected results in the monitoring index.\n\n2. ✅ Verify
privilege-based access for init flow\n\t•\tUsers with the correct Kibana
privileges receive a 200 OK response\n\t•\tUsers without the required
privileges receive a 403 Forbidden\n\n---------\n\nCo-authored-by: Tiago
Vila Verde <[email protected]>\nCo-authored-by: kibanamachine
<[email protected]>\nCo-authored-by: Tiago
Vila Verde <[email protected]>\nCo-authored-by:
natasha-moore-elastic
<[email protected]>\nCo-authored-by:
Mark Hopkin <[email protected]>\nCo-authored-by: machadoum
<[email protected]>","sha":"746b4c86024be146c3e3579057235025126aa687","branchLabelMapping":{"^v9.2.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","Team:
SecuritySolution","Theme: entity_analytics","Feature:Entity
Analytics","Team:Entity
Analytics","backport:version","v9.1.0","v9.2.0"],"title":"[Privmon] Add
Functional Tests for Index Sync and init API
Access","number":229137,"url":"https://github.com/elastic/kibana/pull/229137","mergeCommit":{"message":"[Privmon]
Add Functional Tests for Index Sync and init API Access
(elastic#229137)\n\nRebased with this
PR:\nhttps://github.com/elastic/pull/229019/commits\n\n##
Summary\nThis PR introduces some basic FTR tests for the privilege
monitor\nsyncing.\n1. ✅ Plain Index Sync Test - Verifies that the
Privileged Monitoring\nsystem can correctly:\n\t•\tSync user data from a
plain (non-integration) index,\n\t•\tDeduplicate users,\n\t•\tPersist
the expected results in the monitoring index.\n\n2. ✅ Verify
privilege-based access for init flow\n\t•\tUsers with the correct Kibana
privileges receive a 200 OK response\n\t•\tUsers without the required
privileges receive a 403 Forbidden\n\n---------\n\nCo-authored-by: Tiago
Vila Verde <[email protected]>\nCo-authored-by: kibanamachine
<[email protected]>\nCo-authored-by: Tiago
Vila Verde <[email protected]>\nCo-authored-by:
natasha-moore-elastic
<[email protected]>\nCo-authored-by:
Mark Hopkin <[email protected]>\nCo-authored-by: machadoum
<[email protected]>","sha":"746b4c86024be146c3e3579057235025126aa687"}},"sourceBranch":"main","suggestedTargetBranches":["9.1"],"targetPullRequestStates":[{"branch":"9.1","label":"v9.1.0","branchLabelMappingKey":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"main","label":"v9.2.0","branchLabelMappingKey":"^v9.2.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/229137","number":229137,"mergeCommit":{"message":"[Privmon]
Add Functional Tests for Index Sync and init API Access
(elastic#229137)\n\nRebased with this
PR:\nhttps://github.com/elastic/pull/229019/commits\n\n##
Summary\nThis PR introduces some basic FTR tests for the privilege
monitor\nsyncing.\n1. ✅ Plain Index Sync Test - Verifies that the
Privileged Monitoring\nsystem can correctly:\n\t•\tSync user data from a
plain (non-integration) index,\n\t•\tDeduplicate users,\n\t•\tPersist
the expected results in the monitoring index.\n\n2. ✅ Verify
privilege-based access for init flow\n\t•\tUsers with the correct Kibana
privileges receive a 200 OK response\n\t•\tUsers without the required
privileges receive a 403 Forbidden\n\n---------\n\nCo-authored-by: Tiago
Vila Verde <[email protected]>\nCo-authored-by: kibanamachine
<[email protected]>\nCo-authored-by: Tiago
Vila Verde <[email protected]>\nCo-authored-by:
natasha-moore-elastic
<[email protected]>\nCo-authored-by:
Mark Hopkin <[email protected]>\nCo-authored-by: machadoum
<[email protected]>","sha":"746b4c86024be146c3e3579057235025126aa687"}}]}]
BACKPORT-->

Co-authored-by: Charlotte Alexandra Wilson <[email protected]>
Co-authored-by: Tiago Vila Verde <[email protected]>
Co-authored-by: Tiago Vila Verde <[email protected]>
Co-authored-by: natasha-moore-elastic <[email protected]>
Co-authored-by: Mark Hopkin <[email protected]>
Co-authored-by: machadoum <[email protected]>

Author: 6 people

x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/privilege_monitoring/constants.ts

@@ -6,6 +6,7 @@
  */
 
 import { EXCLUDE_ELASTIC_CLOUD_INDICES, INCLUDE_INDEX_PATTERN } from '../../../../common/constants';
+
 export const SCOPE = ['securitySolution'];
 export const TYPE = 'entity_analytics:monitoring:privileges:engine';
 export const VERSION = '1.0.0';

x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/privilege_monitoring/privilege_monitoring_data_client.ts

@@ -5,14 +5,14 @@
  * 2.0.
  */
 
-import type {
-  Logger,
-  ElasticsearchClient,
-  SavedObjectsClientContract,
-  AuditLogger,
-  IScopedClusterClient,
-  AnalyticsServiceSetup,
-  AuditEvent,
+import {
+  type Logger,
+  type ElasticsearchClient,
+  type SavedObjectsClientContract,
+  type AuditLogger,
+  type IScopedClusterClient,
+  type AnalyticsServiceSetup,
+  type AuditEvent,
 } from '@kbn/core/server';
 
 import type { TaskManagerStartContract } from '@kbn/task-manager-plugin/server';
@@ -81,6 +81,8 @@ import {
   eventIngestPipeline,
 } from './elasticsearch/pipelines/event_ingested';
 import type { BulkProcessingResults } from './users/bulk/types';
+import { ignoreSONotFoundError } from './saved_objects/helpers';
+
 interface PrivilegeMonitoringClientOpts {
   logger: Logger;
   clusterClient: IScopedClusterClient;
@@ -146,7 +148,6 @@ export class PrivilegeMonitoringDataClient {
       if (this.apiKeyGenerator) {
         await this.apiKeyGenerator.generate();
       }
-
       await startPrivilegeMonitoringTask({
         logger: this.opts.logger,
         namespace: this.opts.namespace,
@@ -180,14 +181,13 @@ export class PrivilegeMonitoringDataClient {
         },
       });
     }
-
     return descriptor;
   }
 
   async delete(deleteData = false): Promise<{ deleted: boolean }> {
     this.log('info', 'Deleting privilege monitoring engine');
 
-    await this.engineClient.delete();
+    await this.engineClient.delete().catch(ignoreSONotFoundError);
 
     if (deleteData) {
       await this.esClient.indices.delete(
@@ -208,9 +208,11 @@ export class PrivilegeMonitoringDataClient {
       taskManager: this.opts.taskManager,
     });
 
-    await this.monitoringIndexSourceClient
-      .findAll({})
-      .then((sos) => sos.forEach((so) => this.monitoringIndexSourceClient.delete(so.id)));
+    const allDataSources = await this.monitoringIndexSourceClient.findAll({});
+    const deleteSourcePromises = allDataSources.map((so) =>
+      this.monitoringIndexSourceClient.delete(so.id)
+    );
+    await Promise.all(deleteSourcePromises);
 
     return { deleted: true };
   }
@@ -789,8 +791,8 @@ export class PrivilegeMonitoringDataClient {
   }
 
   private createOrUpdateDefaultDataSource = async () => {
-    const sourceName = `default-monitoring-index-${this.opts.namespace}`;
-
+    // const sourceName = `default-monitoring-index-${this.opts.namespace}`;
+    const sourceName = this.getIndex(); // `entity_analytics.monitoring.users-${this.opts.namespace}`;
     const defaultIndexSource: CreateMonitoringEntitySource = {
       type: 'index',
       managed: true,

x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/privilege_monitoring/routes/monitoring_entity_source/monitoring_entity_source.ts

@@ -64,19 +64,16 @@ export const monitoringEntitySourceRoute = (
           const secSol = await context.securitySolution;
           const client = secSol.getMonitoringEntitySourceDataClient();
 
+          const body = await client.init(request.body);
           const privMonDataClient = await secSol.getPrivilegeMonitoringDataClient();
           const engineStatus = await privMonDataClient.getEngineStatus();
-
           try {
             if (engineStatus.status === PRIVILEGE_MONITORING_ENGINE_STATUS.STARTED) {
               await privMonDataClient.scheduleNow();
             }
           } catch (e) {
             logger.warn(`[Privilege Monitoring] Error scheduling task, received ${e.message}`);
           }
-
-          const body = await client.init(request.body);
-
           return response.ok({ body });
         } catch (e) {
           const error = transformError(e);
@@ -88,7 +85,6 @@ export const monitoringEntitySourceRoute = (
         }
       }
     );
-
   router.versioned
     .get({
       access: 'public',

x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/privilege_monitoring/saved_objects/helpers.ts

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { SavedObjectsErrorHelpers } from '@kbn/core/server';
+
+export const ignoreSONotFoundError = (error: Error) => {
+  if (!SavedObjectsErrorHelpers.isNotFoundError(error)) {
+    throw error;
+  }
+};

x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/privilege_monitoring/saved_objects/monitoring_entity_source.ts

@@ -49,7 +49,6 @@ export class MonitoringEntitySourceDescriptorClient {
     const scopedSoClient = this.dependencies.soClient.asScopedToNamespace(
       this.dependencies.namespace
     );
-
     return scopedSoClient.find<MonitoringEntitySource>({
       type: monitoringEntitySourceTypeName,
       filter: this.getQueryFilters(query),

x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/privilege_monitoring/saved_objects/privilege_monitoring.test.ts

@@ -52,7 +52,7 @@ describe('PrivilegeMonitoringEngineDescriptorClient', () => {
     expect(soClient.create).toHaveBeenCalledWith(
       privilegeMonitoringTypeName,
       { status: PRIVILEGE_MONITORING_ENGINE_STATUS.STARTED },
-      { id: `privilege-monitoring-${namespace}` }
+      { id: `privilege-monitoring-${namespace}`, refresh: 'wait_for' }
     );
     expect(result).toEqual({ status: 'installing' });
   });
@@ -161,7 +161,8 @@ describe('PrivilegeMonitoringEngineDescriptorClient', () => {
 
     expect(soClient.delete).toHaveBeenCalledWith(
       privilegeMonitoringTypeName,
-      `privilege-monitoring-${namespace}`
+      `privilege-monitoring-${namespace}`,
+      { refresh: 'wait_for' }
     );
   });
 });

x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/privilege_monitoring/saved_objects/privilege_monitoring.ts

@@ -40,7 +40,7 @@ export class PrivilegeMonitoringEngineDescriptorClient {
       {
         status: PRIVILEGE_MONITORING_ENGINE_STATUS.STARTED,
       },
-      { id: this.getSavedObjectId() }
+      { id: this.getSavedObjectId(), refresh: 'wait_for' }
     );
     return attributes;
   }
@@ -97,6 +97,6 @@ export class PrivilegeMonitoringEngineDescriptorClient {
 
   async delete() {
     const id = this.getSavedObjectId();
-    return this.deps.soClient.delete(privilegeMonitoringTypeName, id);
+    return this.deps.soClient.delete(privilegeMonitoringTypeName, id, { refresh: 'wait_for' });
   }
 }

x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/privilege_monitoring/tasks/privilege_monitoring_task.ts

@@ -194,7 +194,6 @@ export const startPrivilegeMonitoringTask = async ({
   taskManager,
 }: StartParams) => {
   const taskId = getTaskId(namespace);
-
   try {
     await taskManager.ensureScheduled({
       id: taskId,

x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/types.ts

@@ -18,7 +18,6 @@ import type { ConfigType } from '../../config';
 import type { StartPlugins } from '../../plugin';
 import type { SecuritySolutionPluginRouter } from '../../types';
 export type EntityAnalyticsConfig = ConfigType['entityAnalytics'];
-
 export interface EntityAnalyticsRoutesDeps {
   router: SecuritySolutionPluginRouter;
   logger: Logger;

x-pack/test/security_solution_api_integration/test_suites/entity_analytics/monitoring/trial_license_complete_tier/engine.ts

@@ -5,39 +5,71 @@
  * 2.0.
  */
 
-import expect from '@kbn/expect';
+import expect from 'expect';
 import { FtrProviderContext } from '../../../../ftr_provider_context';
-import { dataViewRouteHelpersFactory } from '../../utils/data_view';
-import { enablePrivmonSetting } from '../../utils';
+import { disablePrivmonSetting, enablePrivmonSetting } from '../../utils';
+import { PrivMonUtils } from './privileged_users/utils';
 
 export default ({ getService }: FtrProviderContext) => {
   const api = getService('securitySolutionApi');
-  const supertest = getService('supertest');
+  const kibanaServer = getService('kibanaServer');
+  const privMonUtils = PrivMonUtils(getService);
   const log = getService('log');
+  const es = getService('es');
+  const retry = getService('retry');
 
-  describe('@ess @serverless @skipInServerlessMKI Entity Privilege Monitoring APIs', () => {
-    const dataView = dataViewRouteHelpersFactory(supertest);
-    const kibanaServer = getService('kibanaServer');
+  const createUserIndex = async (indexName: string) =>
+    es.indices.create({
+      index: indexName,
+      mappings: {
+        properties: {
+          user: {
+            properties: {
+              name: {
+                type: 'keyword',
+                fields: {
+                  text: { type: 'text' },
+                },
+              },
+              role: {
+                type: 'keyword',
+              },
+            },
+          },
+        },
+      },
+    });
+
+  const waitForPrivMonUsersToBeSynced = async (length = 1) =>
+    retry.waitForWithTimeout('Wait for PrivMon users to be synced', 90000, async () => {
+      const res = await api.listPrivMonUsers({ query: {} });
+      log.info(`PrivMon users sync check: found ${res.body.length} users`);
+      return res.body.length >= length; // wait until we have at least one user
+    });
+
+  describe('@ess Entity Privilege Monitoring APIs', () => {
     before(async () => {
-      await dataView.create('security-solution');
       await enablePrivmonSetting(kibanaServer);
     });
 
     after(async () => {
-      await dataView.delete('security-solution');
+      await disablePrivmonSetting(kibanaServer);
+    });
+
+    afterEach(async () => {
+      await api.deleteMonitoringEngine({ query: { data: true } });
     });
 
     describe('health', () => {
       it('should be healthy', async () => {
-        log.info(`Checking health of privilege monitoring`);
         const res = await api.privMonHealth();
 
         if (res.status !== 200) {
           log.error(`Health check failed`);
           log.error(JSON.stringify(res.body));
         }
 
-        expect(res.status).eql(200);
+        expect(res.status).toEqual(200);
       });
     });
 
@@ -51,7 +83,7 @@ export default ({ getService }: FtrProviderContext) => {
           log.error(JSON.stringify(res1.body));
         }
 
-        expect(res1.status).eql(200);
+        expect(res1.status).toEqual(200);
 
         log.info(`Re-initializing Privilege Monitoring engine`);
         const res2 = await api.initMonitoringEngine();
@@ -60,7 +92,78 @@ export default ({ getService }: FtrProviderContext) => {
           log.error(JSON.stringify(res2.body));
         }
 
-        expect(res2.status).eql(200);
+        expect(res2.status).toEqual(200);
+      });
+    });
+
+    describe('plain index sync', () => {
+      const indexName = 'tatooine-privileged-users';
+      const entitySource = {
+        type: 'index',
+        name: 'StarWars',
+        managed: true,
+        indexPattern: indexName,
+        enabled: true,
+        matchers: [
+          {
+            fields: ['user.role'],
+            values: ['admin'],
+          },
+        ],
+        filter: {},
+      };
+
+      beforeEach(async () => {
+        await createUserIndex(indexName);
+      });
+
+      afterEach(async () => {
+        try {
+          await es.indices.delete({ index: indexName }, { ignore: [404] });
+        } catch (err) {
+          log.warning(`Failed to clean up in afterEach: ${err.message}`);
+        }
+      });
+
+      it('should sync plain index', async () => {
+        // Bulk insert documents
+        const uniqueUsers = [
+          'Luke Skywalker',
+          'Leia Organa',
+          'Han Solo',
+          'Chewbacca',
+          'Obi-Wan Kenobi',
+          'Yoda',
+          'R2-D2',
+          'C-3PO',
+          'Darth Vader',
+        ].flatMap((name) => [{ index: {} }, { user: { name, role: 'admin' } }]);
+        const repeatedUsers = Array.from({ length: 150 }).flatMap(() => [
+          { index: {} },
+          { user: { name: 'C-3PO', role: 'admin' } },
+        ]);
+
+        const bulkBody = [...uniqueUsers, ...repeatedUsers];
+        await es.bulk({ index: indexName, body: bulkBody, refresh: true });
+
+        // Call init to trigger the sync
+        await privMonUtils.initPrivMonEngine();
+
+        // register entity source
+        const response = await api.createEntitySource({ body: entitySource });
+        expect(response.status).toBe(200);
+
+        // default-monitoring-index should exist now
+        const sources = await api.listEntitySources({ query: {} });
+        const names = sources.body.map((s: any) => s.name);
+        expect(names).toContain('StarWars');
+        await waitForPrivMonUsersToBeSynced(9);
+        // Check if the users are indexed
+        const res = await api.listPrivMonUsers({ query: {} });
+        const userNames = res.body.map((u: any) => u.user.name);
+        expect(userNames).toContain('Luke Skywalker');
+        expect(userNames).toContain('C-3PO');
+        expect(userNames.filter((name: string) => name === 'C-3PO')).toHaveLength(1);
       });
     });
   });