[SecuritySolution] Add index privileges check to applyDataViewIndices (#214803)

## Summary

Add a new privileges check before executing `applyDataViewIndices`.

This change impacts the API call `applyDataViewIndices` and the job. 
`applyDataViewIndices` updates the transforms. Executing without
privileges generates a silence error because the transform can't run.

I also added some extra unit tests for `applyDataViewIndices`.

Required privileges
['read', 'view_index_metadata'] for all security solution dataview +
asset_criticality and risk_score indices.


### How to test it
1. **API call with unprivileged user scenario**
* Enable the entity store with a superuser
* Create an unprivileged user
* Call `POST kbn:api/entity_store/engines/apply_dataview_indices`
* It should return an error
* Add the required privileges
* It executes successfully

2. **Task execution with an unprivileged user scenario**
* Create a user and add privileges only for the required Entity Store
indices
* Login with the new user
* Enable the entity store
* Add a new index to the security data view (the new user shouldn't have
access to the new index)
* Wait for 30min for the job to run, or update the [source
code](https://github.com/elastic/kibana/blob/8d0feb580f13cb1571beaf84a6d5763197211106/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/tasks/data_view_refresh/data_view_refresh_task.ts#L150)
to make it run more often
* The job execution should fail with an error message containing the new
index name.




### Checklist

Reviewers should verify this PR satisfies this list as well.

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

Author: machadoum

x-pack/solutions/security/plugins/security_solution/common/entity_analytics/entity_store/constants.ts

@@ -21,5 +21,8 @@ export const ENTITY_STORE_REQUIRED_ES_CLUSTER_PRIVILEGES = [
   'manage_enrich',
 ];
 
+// Privileges required for the transform to run
+export const ENTITY_STORE_SOURCE_REQUIRED_ES_INDEX_PRIVILEGES = ['read', 'view_index_metadata'];
+
 // The index pattern for the entity store has to support '.entities.v1.latest.noop' index
 export const ENTITY_STORE_INDEX_PATTERN = '.entities.v1.latest.*';

x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.test.ts

@@ -22,6 +22,10 @@ import { EntityType } from '../../../../common/search_strategy';
 import type { InitEntityEngineResponse } from '../../../../common/api/entity_analytics';
 import type { TaskManagerStartContract } from '@kbn/task-manager-plugin/server';
 import { defaultOptions } from './constants';
+import type { SecurityPluginStart } from '@kbn/security-plugin/server';
+import type { KibanaRequest } from '@kbn/core/server';
+import { dataViewPluginMocks } from '@kbn/data-views-plugin/public/mocks';
+import { createStubDataView } from '@kbn/data-views-plugin/common/mocks';
 
 const definition: EntityDefinition = convertToEntityManagerDefinition(
   {
@@ -46,22 +50,73 @@ const definition: EntityDefinition = convertToEntityManagerDefinition(
   { namespace: 'test', filter: '' }
 );
 
+const stubSecurityDataView = createStubDataView({
+  spec: {
+    id: 'security',
+    title: 'security',
+  },
+});
+
+const dataviewService = {
+  ...dataViewPluginMocks.createStartContract(),
+  get: () => Promise.resolve(stubSecurityDataView),
+  clearInstanceCache: () => Promise.resolve(),
+};
+
+const mockGetEntityDefinition = jest.fn().mockResolvedValue([]);
+const mockUpdateEntityDefinition = jest.fn().mockResolvedValue(undefined);
+jest.mock('@kbn/entityManager-plugin/server/lib/entity_client', () => {
+  return {
+    EntityClient: jest.fn().mockImplementation(() => ({
+      updateEntityDefinition: mockUpdateEntityDefinition,
+      getEntityDefinitions: mockGetEntityDefinition,
+    })),
+  };
+});
+
+const mockListDescriptor = jest.fn().mockResolvedValue({ engines: [] });
+const mockUpdateStatus = jest.fn().mockResolvedValue({});
+jest.mock('./saved_object/engine_descriptor', () => {
+  return {
+    EngineDescriptorClient: jest.fn().mockImplementation(() => ({
+      list: mockListDescriptor,
+      updateStatus: mockUpdateStatus,
+    })),
+  };
+});
+
+const mockCheckPrivileges = jest.fn().mockReturnValue({
+  hasAllRequested: true,
+  privileges: {
+    elasticsearch: { cluster: [], index: [] },
+    kibana: [],
+  },
+});
+
 describe('EntityStoreDataClient', () => {
-  const mockSavedObjectClient = savedObjectsClientMock.create();
   const clusterClientMock = elasticsearchServiceMock.createScopedClusterClient();
   const esClientMock = clusterClientMock.asCurrentUser;
   const loggerMock = loggingSystemMock.createLogger();
   const dataClient = new EntityStoreDataClient({
     clusterClient: clusterClientMock,
     logger: loggerMock,
     namespace: 'default',
-    soClient: mockSavedObjectClient,
+    soClient: savedObjectsClientMock.create(),
     kibanaVersion: '9.0.0',
-    dataViewsService: {} as DataViewsService,
-    appClient: {} as AppClient,
+    dataViewsService: dataviewService as unknown as DataViewsService,
+    appClient: {
+      getSourcererDataViewId: jest.fn().mockReturnValue('security-solution'),
+      getAlertsIndex: jest.fn().mockReturnValue('alerts'),
+    } as unknown as AppClient,
     config: {} as EntityStoreConfig,
     experimentalFeatures: mockGlobalState.app.enableExperimental,
     taskManager: {} as TaskManagerStartContract,
+    security: {
+      authz: {
+        checkPrivilegesDynamicallyWithRequest: () => mockCheckPrivileges,
+      },
+    } as unknown as SecurityPluginStart,
+    request: {} as KibanaRequest,
   });
 
   const defaultSearchParams = {
@@ -89,7 +144,7 @@ describe('EntityStoreDataClient', () => {
 
   describe('search entities', () => {
     beforeEach(() => {
-      jest.resetAllMocks();
+      jest.clearAllMocks();
       esClientMock.search.mockResolvedValue(emptySearchResponse);
     });
 
@@ -349,7 +404,7 @@ describe('EntityStoreDataClient', () => {
     let spyInit: jest.SpyInstance;
 
     beforeEach(() => {
-      jest.resetAllMocks();
+      jest.clearAllMocks();
       spyInit = jest
         .spyOn(dataClient, 'init')
         .mockImplementation(() => Promise.resolve({} as InitEntityEngineResponse));
@@ -364,4 +419,84 @@ describe('EntityStoreDataClient', () => {
       expect(spyInit).toHaveBeenCalledWith(EntityType.host, expect.anything(), expect.anything());
     });
   });
+
+  describe('applyDataViewIndices', () => {
+    beforeEach(() => {
+      mockUpdateEntityDefinition.mockClear();
+      jest.clearAllMocks();
+    });
+
+    it('applies data view indices to the entity store', async () => {
+      mockListDescriptor.mockResolvedValueOnce({ engines: [{}] });
+      mockGetEntityDefinition.mockResolvedValueOnce({
+        definitions: [definition],
+      });
+
+      const response = await dataClient.applyDataViewIndices();
+
+      expect(mockUpdateEntityDefinition).toHaveBeenCalled();
+      expect(response.errors.length).toBe(0);
+      expect(response.successes.length).toBe(1);
+    });
+
+    it('returns empty successes and errors if no engines found', async () => {
+      mockListDescriptor.mockResolvedValueOnce({ engines: [] });
+
+      const response = await dataClient.applyDataViewIndices();
+
+      expect(response.successes.length).toBe(0);
+      expect(response.errors.length).toBe(0);
+    });
+
+    it('throws an error if the user does not have required privileges', async () => {
+      mockCheckPrivileges.mockReturnValueOnce({
+        hasAllRequested: false,
+        privileges: {
+          elasticsearch: { cluster: [], index: [] },
+          kibana: [],
+        },
+      });
+
+      mockListDescriptor.mockResolvedValueOnce({ engines: [{}] });
+
+      await expect(dataClient.applyDataViewIndices()).rejects.toThrow(
+        /The current user does not have the required indices privileges.*/
+      );
+    });
+
+    it('skips update if index patterns are the same', async () => {
+      mockListDescriptor.mockResolvedValueOnce({ engines: [{}] });
+      mockGetEntityDefinition.mockResolvedValueOnce({
+        definitions: [
+          {
+            indexPatterns: [
+              stubSecurityDataView.getIndexPattern(),
+              '.asset-criticality.asset-criticality-default',
+              'risk-score.risk-score-latest-default',
+            ],
+          },
+        ],
+      });
+
+      const response = await dataClient.applyDataViewIndices();
+
+      expect(mockUpdateEntityDefinition).not.toHaveBeenCalled();
+      expect(response.successes.length).toBe(1);
+      expect(response.errors.length).toBe(0);
+    });
+
+    it('handles errors during update', async () => {
+      const testErrorMessages = 'Update failed';
+      mockUpdateEntityDefinition.mockRejectedValueOnce(new Error(testErrorMessages));
+      mockListDescriptor.mockResolvedValueOnce({ engines: [{}] });
+      mockGetEntityDefinition.mockResolvedValueOnce({
+        definitions: [definition],
+      });
+
+      const response = await dataClient.applyDataViewIndices();
+
+      expect(response.errors.length).toBeGreaterThan(0);
+      expect(response.errors[0].message).toBe(testErrorMessages);
+    });
+  });
 });

x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts

@@ -13,7 +13,9 @@ import type {
   IScopedClusterClient,
   AuditEvent,
   AnalyticsServiceSetup,
+  KibanaRequest,
 } from '@kbn/core/server';
+import type { SecurityPluginStart } from '@kbn/security-plugin/server';
 import { EntityClient } from '@kbn/entityManager-plugin/server/lib/entity_client';
 import type { HealthStatus, SortOrder } from '@elastic/elasticsearch/lib/api/types';
 import type { TaskManagerStartContract } from '@kbn/task-manager-plugin/server';
@@ -23,6 +25,7 @@ import moment from 'moment';
 import type { EntityDefinitionWithState } from '@kbn/entityManager-plugin/server/lib/entities/types';
 import type { EntityDefinition } from '@kbn/entities-schema';
 import type { estypes } from '@elastic/elasticsearch';
+import { getAllMissingPrivileges } from '../../../../common/entity_analytics/privileges';
 import { merge } from '../../../../common/utils/objects/merge';
 import { getEnabledStoreEntityTypes } from '../../../../common/entity_analytics/entity_store/utils';
 import { EntityType } from '../../../../common/entity_analytics/types';
@@ -96,7 +99,7 @@ import {
 import { CRITICALITY_VALUES } from '../asset_criticality/constants';
 import { createEngineDescription } from './installation/engine_description';
 import { convertToEntityManagerDefinition } from './entity_definitions/entity_manager_conversion';
-
+import { getEntityStoreSourceIndicesPrivileges } from './utils/get_entity_store_privileges';
 import type { ApiKeyManager } from './auth/api_key';
 
 // Workaround. TransformState type is wrong. The health type should be: TransformHealth from '@kbn/transform-plugin/common/types/transform_stats'
@@ -126,6 +129,8 @@ interface EntityStoreClientOpts {
   experimentalFeatures: ExperimentalFeatures;
   telemetry?: AnalyticsServiceSetup;
   apiKeyManager?: ApiKeyManager;
+  security: SecurityPluginStart;
+  request: KibanaRequest;
 }
 
 interface SearchEntitiesParams {
@@ -788,6 +793,41 @@ export class EntityStoreDataClient {
 
     const { engines } = await this.engineClient.list();
 
+    if (engines.length === 0) {
+      logger.debug(
+        `In namespace ${this.options.namespace}: No entity engines found, skipping data view index application`
+      );
+      return {
+        successes: [],
+        errors: [],
+      };
+    }
+
+    const indexPatterns = await buildIndexPatterns(
+      this.options.namespace,
+      this.options.appClient,
+      this.options.dataViewsService
+    );
+
+    const privileges = await getEntityStoreSourceIndicesPrivileges(
+      this.options.request,
+      this.options.security,
+      indexPatterns
+    );
+
+    if (!privileges.has_all_required) {
+      const missingPrivilegesMsg = getAllMissingPrivileges(privileges).elasticsearch.index.map(
+        ({ indexName, privileges: missingPrivileges }) =>
+          `Missing [${missingPrivileges.join(', ')}] privileges for index '${indexName}'.`
+      );
+
+      throw new Error(
+        `The current user does not have the required indices privileges.\n${missingPrivilegesMsg.join(
+          '\n'
+        )}`
+      );
+    }
+
     const updateDefinitionPromises: Array<Promise<EngineDataviewUpdateResult>> = engines.map(
       async (engine) => {
         const originalStatus = engine.status;
@@ -803,12 +843,6 @@ export class EntityStoreDataClient {
           );
         }
 
-        const indexPatterns = await buildIndexPatterns(
-          this.options.namespace,
-          this.options.appClient,
-          this.options.dataViewsService
-        );
-
         // Skip update if index patterns are the same
         if (isEqual(definition.indexPatterns, indexPatterns)) {
           logger.debug(

x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/tasks/data_view_refresh/data_view_refresh_task.ts

@@ -89,7 +89,9 @@ export const registerEntityStoreDataViewRefreshTask = ({
 
     const dataViewsService = await dataViews.dataViewsServiceFactory(soClient, internalUserClient);
 
-    const appClient = appClientFactory.create(await apiKeyManager.getRequestFromApiKey(apiKey));
+    const request = await apiKeyManager.getRequestFromApiKey(apiKey);
+
+    const appClient = appClientFactory.create(request);
 
     const entityStoreClient: EntityStoreDataClient = new EntityStoreDataClient({
       namespace,
@@ -104,6 +106,8 @@ export const registerEntityStoreDataViewRefreshTask = ({
       kibanaVersion,
       dataViewsService,
       config: entityStoreConfig,
+      security,
+      request,
     });
 
     await entityStoreClient.applyDataViewIndices();

x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/utils/get_entity_store_privileges.ts

@@ -12,6 +12,7 @@ import { RISK_SCORE_INDEX_PATTERN } from '../../../../../common/constants';
 import {
   ENTITY_STORE_INDEX_PATTERN,
   ENTITY_STORE_REQUIRED_ES_CLUSTER_PRIVILEGES,
+  ENTITY_STORE_SOURCE_REQUIRED_ES_INDEX_PRIVILEGES,
 } from '../../../../../common/entity_analytics/entity_store/constants';
 import { checkAndFormatPrivileges } from '../../utils/check_and_format_privileges';
 import { entityEngineDescriptorTypeName } from '../saved_object';
@@ -22,13 +23,7 @@ export const getEntityStorePrivileges = (
   securitySolutionIndices: string[]
 ) => {
   // The entity store needs access to all security solution indices
-  const indicesPrivileges = securitySolutionIndices.reduce<Record<string, string[]>>(
-    (acc, index) => {
-      acc[index] = ['read', 'view_index_metadata'];
-      return acc;
-    },
-    {}
-  );
+  const indicesPrivileges = getEntityStoreSourceRequiredIndicesPrivileges(securitySolutionIndices);
 
   // The entity store has to create the following indices
   indicesPrivileges[ENTITY_STORE_INDEX_PATTERN] = ['read', 'manage'];
@@ -49,3 +44,30 @@ export const getEntityStorePrivileges = (
     },
   });
 };
+
+// Get the index privileges required for running the transform
+export const getEntityStoreSourceIndicesPrivileges = (
+  request: KibanaRequest,
+  security: SecurityPluginStart,
+  indexPatterns: string[]
+) => {
+  const requiredIndicesPrivileges = getEntityStoreSourceRequiredIndicesPrivileges(indexPatterns);
+
+  return checkAndFormatPrivileges({
+    request,
+    security,
+    privilegesToCheck: {
+      elasticsearch: {
+        cluster: [],
+        index: requiredIndicesPrivileges,
+      },
+    },
+  });
+};
+
+const getEntityStoreSourceRequiredIndicesPrivileges = (securitySolutionIndices: string[]) => {
+  return securitySolutionIndices.reduce<Record<string, string[]>>((acc, index) => {
+    acc[index] = ENTITY_STORE_SOURCE_REQUIRED_ES_INDEX_PRIVILEGES;
+    return acc;
+  }, {});
+};

x-pack/solutions/security/plugins/security_solution/server/request_context_factory.ts

@@ -266,6 +266,8 @@ export class RequestContextFactory implements IRequestContextFactory {
             request,
             namespace: getSpaceId(),
           }),
+          security: startPlugins.security,
+          request,
         });
       }),
       getAssetInventoryClient: memoize(() => {