[Security Solution][Endpoint] Update response action creation so that it store policy information when action is created (elastic#218175)

## Summary

The following changes are in support of space awareness for response
actions (currently behind feature flag:
`endpointManagementSpaceAwarenessEnabled`). All response actions will
now start to store agent policy information.

- A new property will be stored in the Action Request document that
captures policy information about each agent the action was sent to
(`agent.policy: []`).
- All response action client instances must now be initiated with a
`spaceId`
- A new method was added to the internal Fleet Services to retrieve all
`namespace`'s in use by a given integration type in the active space
- The SentinelOne and Microsoft run host scripts were enhaced so that
the host VM name includes the space id

Author: paul-tavares

x-pack/solutions/security/plugins/security_solution/common/endpoint/data_generators/endpoint_action_generator.ts

@@ -50,10 +50,22 @@ export class EndpointActionGenerator extends BaseDataGenerator {
     overrides: DeepPartial<LogsEndpointAction<TParameters, TOutputContent, TMeta>> = {}
   ): LogsEndpointAction<TParameters, TOutputContent, TMeta> {
     const timeStamp = overrides['@timestamp'] ? new Date(overrides['@timestamp']) : new Date();
+    const agent = (overrides.agent?.id ?? [
+      this.seededUUIDv4(),
+    ]) as LogsEndpointAction['agent']['id'];
+    const agentId = Array.isArray(agent) ? (agent[0] as string) : agent;
     const doc: LogsEndpointAction<TParameters, TOutputContent, TMeta> = {
       '@timestamp': timeStamp.toISOString(),
       agent: {
-        id: [this.seededUUIDv4()],
+        id: agent,
+        policy: [
+          {
+            agentId,
+            elasticAgentId: agentId,
+            integrationPolicyId: 'integration-policy-1',
+            agentPolicyId: 'agent-policy-1',
+          },
+        ],
       },
       EndpointActions: {
         action_id: this.seededUUIDv4(),

x-pack/solutions/security/plugins/security_solution/common/endpoint/data_generators/microsoft_defender_data_generator.ts

@@ -5,141 +5,167 @@
  * 2.0.
  */
 
+import type { DeepPartial } from 'utility-types';
+import { merge } from 'lodash';
+import type { SearchHit } from '@elastic/elasticsearch/lib/api/types';
+import { buildIndexNameWithNamespace } from '../utils/index_name_utilities';
+import { MICROSOFT_DEFENDER_ENDPOINT_LOG_INDEX_PATTERN } from '../service/response_actions/microsoft_defender';
 import { BaseDataGenerator } from './base_data_generator';
+import type { MicrosoftDefenderEndpointLogEsDoc } from '../types';
 
 export class MicrosoftDefenderDataGenerator extends BaseDataGenerator {
   /**
    * Generates a MS Defender endpoint log as ingested by the Microsoft Defender for Endpoint
    * integration into the `logs-microsoft_defender_endpoint.log` index.
    * If adding this generated document to ES, make sure that the integration has been installed.
    */
-  generateEndpointLog() {
+  generateEndpointLog(
+    overrides: DeepPartial<MicrosoftDefenderEndpointLogEsDoc> = {}
+  ): MicrosoftDefenderEndpointLogEsDoc {
     const now = new Date().toISOString();
 
-    return {
-      agent: {
-        name: 'ptavares-agentless-integrations-default-8511',
-        id: 'a572fc2e-0276-494a-b693-6e907bc2a78b',
-        ephemeral_id: 'e7d70430-d25e-4d72-863e-918ac36bbbf7',
-        type: 'filebeat',
-        version: '9.0.0',
-      },
-      process: {
-        parent: {
+    return merge(
+      {
+        agent: {
+          name: 'ptavares-agentless-integrations-default-8511',
+          id: 'a572fc2e-0276-494a-b693-6e907bc2a78b',
+          ephemeral_id: 'e7d70430-d25e-4d72-863e-918ac36bbbf7',
+          type: 'filebeat',
+          version: '9.0.0',
+        },
+        process: {
+          parent: {
+            start: now,
+            pid: 9901,
+          },
           start: now,
-          pid: 9901,
+          pid: 10083,
+          command_line: '-bash',
         },
-        start: now,
-        pid: 10083,
-        command_line: '-bash',
-      },
-      elastic_agent: {
-        id: 'a572fc2e-0276-494a-b693-6e907bc2a78b',
-        version: '9.0.0',
-        snapshot: true,
-      },
-      rule: {
-        description:
-          'Remote file transfer activity was observed on this device. Attackers might be attempting to steal data from the device or move laterally on the network.',
-      },
-      message: 'Remote exfiltration activity',
-      microsoft: {
-        defender_endpoint: {
-          evidence: {
-            accountName: 'ubuntu',
-            detectionStatus: 'Detected',
-            parentProcessFileName: 'bash',
-            entityType: 'Process',
-            evidenceCreationTime: now,
-            domainName: 'discerning-spaniel',
+        elastic_agent: {
+          id: 'a572fc2e-0276-494a-b693-6e907bc2a78b',
+          version: '9.0.0',
+          snapshot: true,
+        },
+        rule: {
+          description:
+            'Remote file transfer activity was observed on this device. Attackers might be attempting to steal data from the device or move laterally on the network.',
+        },
+        message: 'Remote exfiltration activity',
+        microsoft: {
+          defender_endpoint: {
+            evidence: {
+              accountName: 'ubuntu',
+              detectionStatus: 'Detected',
+              parentProcessFileName: 'bash',
+              entityType: 'Process',
+              evidenceCreationTime: now,
+              domainName: 'discerning-spaniel',
+            },
+            mitreTechniques: [
+              'T1005',
+              'T1020',
+              'T1041',
+              'T1048',
+              'T1071',
+              'T1071.001',
+              'T1204.001',
+              'T1567',
+              'T1570',
+            ],
+            detectorId: this.seededUUIDv4(),
+            investigationState: 'UnsupportedOs',
+            incidentId: '4',
+            lastUpdateTime: now,
+            status: 'New',
           },
-          mitreTechniques: [
-            'T1005',
-            'T1020',
-            'T1041',
-            'T1048',
-            'T1071',
-            'T1071.001',
-            'T1204.001',
-            'T1567',
-            'T1570',
+        },
+        tags: ['microsoft-defender-endpoint', 'forwarded'],
+        cloud: {
+          instance: {
+            id: '7bcf55e03728756dbf02ba7979a0c6218321ade7',
+          },
+          provider: 'azure',
+          account: {
+            id: 'c38d90f4-369c-4815-ab69-4663a1f5c115',
+          },
+        },
+        input: {
+          type: 'httpjson',
+        },
+        observer: {
+          product: 'Defender for Endpoint',
+          vendor: 'Microsoft',
+          name: 'WindowsDefenderAtp',
+        },
+        '@timestamp': now,
+        file: {
+          path: '/usr/bin/',
+          name: 'bash',
+          hash: {
+            sha1: 'ce4fbd66c02e235bbc8dfa4a512c51414d8e0e67',
+            sha256: 'c5f8a98c674631609902846fae6df219b3b16d97db58cf1c1334f8eb14962bde',
+          },
+        },
+        ecs: {
+          version: '8.11.0',
+        },
+        related: {
+          hosts: ['discerning-spaniel'],
+          hash: [
+            'ce4fbd66c02e235bbc8dfa4a512c51414d8e0e67',
+            'c5f8a98c674631609902846fae6df219b3b16d97db58cf1c1334f8eb14962bde',
           ],
-          detectorId: this.seededUUIDv4(),
-          investigationState: 'UnsupportedOs',
-          incidentId: '4',
-          lastUpdateTime: now,
-          status: 'New',
         },
-      },
-      tags: ['microsoft-defender-endpoint', 'forwarded'],
-      cloud: {
-        instance: {
-          id: '7bcf55e03728756dbf02ba7979a0c6218321ade7',
+        data_stream: {
+          namespace: 'default',
+          type: 'logs',
+          dataset: 'microsoft_defender_endpoint.log',
         },
-        provider: 'azure',
-        account: {
-          id: 'c38d90f4-369c-4815-ab69-4663a1f5c115',
+        host: {
+          hostname: 'discerning-spaniel',
+          name: 'discerning-spaniel',
         },
-      },
-      input: {
-        type: 'httpjson',
-      },
-      observer: {
-        product: 'Defender for Endpoint',
-        vendor: 'Microsoft',
-        name: 'WindowsDefenderAtp',
-      },
-      '@timestamp': now,
-      file: {
-        path: '/usr/bin/',
-        name: 'bash',
-        hash: {
-          sha1: 'ce4fbd66c02e235bbc8dfa4a512c51414d8e0e67',
-          sha256: 'c5f8a98c674631609902846fae6df219b3b16d97db58cf1c1334f8eb14962bde',
+        threat: {
+          framework: 'MITRE ATT&CK',
+          technique: {
+            name: ['Exfiltration'],
+          },
         },
-      },
-      ecs: {
-        version: '8.11.0',
-      },
-      related: {
-        hosts: ['discerning-spaniel'],
-        hash: [
-          'ce4fbd66c02e235bbc8dfa4a512c51414d8e0e67',
-          'c5f8a98c674631609902846fae6df219b3b16d97db58cf1c1334f8eb14962bde',
-        ],
-      },
-      data_stream: {
-        namespace: 'default',
-        type: 'logs',
-        dataset: 'microsoft_defender_endpoint.log',
-      },
-      host: {
-        hostname: 'discerning-spaniel',
-        name: 'discerning-spaniel',
-      },
-      threat: {
-        framework: 'MITRE ATT&CK',
-        technique: {
-          name: ['Exfiltration'],
+        event: {
+          severity: 3,
+          created: now,
+          kind: 'alert',
+          timezone: 'UTC',
+          start: now,
+          type: ['start'],
+          duration: 5253721000,
+          agent_id_status: 'verified',
+          ingested: now,
+          provider: 'defender_endpoint',
+          action: 'Exfiltration',
+          end: now,
+          id: this.seededUUIDv4(),
+          category: ['host', 'process'],
+          dataset: 'microsoft_defender_endpoint.log',
         },
       },
-      event: {
-        severity: 3,
-        created: now,
-        kind: 'alert',
-        timezone: 'UTC',
-        start: now,
-        type: ['start'],
-        duration: 5253721000,
-        agent_id_status: 'verified',
-        ingested: now,
-        provider: 'defender_endpoint',
-        action: 'Exfiltration',
-        end: now,
-        id: this.seededUUIDv4(),
-        category: ['host', 'process'],
-        dataset: 'microsoft_defender_endpoint.log',
-      },
-    };
+      overrides
+    );
+  }
+
+  generateEndpointLogEsHit(
+    overrides: DeepPartial<MicrosoftDefenderEndpointLogEsDoc> = {}
+  ): SearchHit<MicrosoftDefenderEndpointLogEsDoc> {
+    return this.toEsSearchHit(
+      this.generateEndpointLog(overrides),
+      buildIndexNameWithNamespace(MICROSOFT_DEFENDER_ENDPOINT_LOG_INDEX_PATTERN, 'default')
+    );
+  }
+
+  generateEndpointLogEsSearchResponse(
+    docs: Array<SearchHit<MicrosoftDefenderEndpointLogEsDoc>> = [this.generateEndpointLogEsHit()]
+  ) {
+    return this.toEsSearchResponse(docs);
   }
 }

x-pack/solutions/security/plugins/security_solution/common/endpoint/data_generators/sentinelone_data_generator.ts

@@ -18,6 +18,7 @@ import type {
   SentinelOneGetRemoteScriptStatusApiResponse,
   SentinelOneRemoteScriptExecutionStatus,
 } from '@kbn/stack-connectors-plugin/common/sentinelone/types';
+import { buildIndexNameWithNamespace } from '../utils/index_name_utilities';
 import { EndpointActionGenerator } from './endpoint_action_generator';
 import { SENTINEL_ONE_ACTIVITY_INDEX_PATTERN } from '../..';
 import type {
@@ -26,7 +27,9 @@ import type {
   EndpointActionDataParameterTypes,
   EndpointActionResponseDataOutput,
   SentinelOneActivityDataForType80,
+  SentinelOneAgentEsDoc,
 } from '../types';
+import { SENTINEL_ONE_AGENT_INDEX_PATTERN } from '../service/response_actions/sentinel_one';
 
 export class SentinelOneDataGenerator extends EndpointActionGenerator {
   static readonly scriptExecutionStatusValues: Readonly<
@@ -428,6 +431,45 @@ export class SentinelOneDataGenerator extends EndpointActionGenerator {
       pagination: { totalItems: 1, nextCursor: undefined },
     };
   }
+
+  /**
+   * Generate a SentinelOne Agent record that is ingested into Elasticsearch by the
+   * integration into `logs-sentinel_one.agent-`
+   */
+  generateAgentEsDoc(overrides: DeepPartial<SentinelOneAgentEsDoc> = {}): SentinelOneAgentEsDoc {
+    return merge(
+      {
+        agent: {
+          id: '1-2-3',
+          type: 'filebeat',
+          version: '9.1.0',
+        },
+        sentinel_one: {
+          agent: {
+            agent: {
+              id: 's1-agent-1',
+            },
+          },
+        },
+      },
+      overrides
+    );
+  }
+
+  generateAgentEsSearchHit(
+    overrides: DeepPartial<SentinelOneAgentEsDoc> = {}
+  ): SearchHit<SentinelOneAgentEsDoc> {
+    return this.toEsSearchHit(
+      this.generateAgentEsDoc(overrides),
+      buildIndexNameWithNamespace(SENTINEL_ONE_AGENT_INDEX_PATTERN, 'default')
+    );
+  }
+
+  generateAgentEsSearchResponse(
+    docs: Array<SearchHit<SentinelOneAgentEsDoc>> = [this.generateAgentEsSearchHit()]
+  ): SearchResponse<SentinelOneAgentEsDoc> {
+    return this.toEsSearchResponse<SentinelOneAgentEsDoc>(docs);
+  }
 }
 
 // Activity types from SentinelOne. Values can be retrieved from the SentineOne API at:

x-pack/solutions/security/plugins/security_solution/common/endpoint/service/response_actions/constants.ts

@@ -213,10 +213,10 @@ export const RESPONSE_ACTIONS_ALERT_AGENT_ID_FIELDS: Readonly<
   ],
   crowdstrike: ['device.id'],
   microsoft_defender_endpoint: [
+    'cloud.instance.id',
     'm365_defender.alerts.entities.deviceId',
     'm365_defender.alerts.devices.mdatpDeviceId',
     'm365_defender.incident.alert.evidence.mde_device_id',
-    'cloud.instance.id',
   ],
 });