[Rule Tunings] Reduce Usage of Flattened Fields in AWS Rules (#4892)

* [Rule Tunings] Reduce Usage of Flattened Fields in AWS Rules

This PR is in part a response to the following issues regarding the future of flattened fields in AWS, which we use as an essential part of our ruleset. However, this is also in response to the ongoing ruleset audit. Some of the flattened fields used are not truly necessary for the alert to trigger or can be replaced by a different field. Those changes have been made here and our non_ecs file has been edited to remove the unnecessary fields. Additionally, flattened fields have been removed from highlighted fields, and from investigation guides.

* Update discovery_ec2_userdata_request_for_ec2_instance.toml

updated_date

* Update execution_ssm_sendcommand_by_rare_user.toml

updated_date

* Update non-ecs-schema.json

add necessary field for ModifyInstanceAttribute action

* Update persistence_ec2_security_group_configuration_change_detection.toml

added missing event.action AuthorizeSecurityGroupIngress, narrowed scope for ModifyInstanceAttribute action by adding a necessary flattened_field

* Update privilege_escalation_iam_customer_managed_policy_attached_to_role.toml

updated min_stack_version for new field target.entity.id

* Update privilege_escalation_iam_customer_managed_policy_attached_to_role.toml

* Update privilege_escalation_iam_update_assume_role_policy.toml

updating min_stack to account of target.entity.id field

* Update impact_s3_excessive_object_encryption_with_sse_c.toml

adding highlighted fields

* Update rules/integrations/aws/exfiltration_dynamodb_table_exported_to_s3.toml

* Apply suggestions from code review

---------

Co-authored-by: Mika Ayenson, PhD <[email protected]>
Co-authored-by: Ruben Groenewoud <[email protected]>

(cherry picked from commit b141ebc)

Author: imays11

detection_rules/etc/non-ecs-schema.json

@@ -155,28 +155,18 @@
     "auditd.data.a3": "keyword"
   },
   "logs-aws.cloudtrail-*": {
-    "aws.cloudtrail.flattened.request_parameters.cidrIp": "keyword",
-    "aws.cloudtrail.flattened.request_parameters.fromPort": "keyword",
-    "aws.cloudtrail.flattened.request_parameters.roleArn": "keyword",
-    "aws.cloudtrail.flattened.request_parameters.roleName": "keyword",
-    "aws.cloudtrail.flattened.request_parameters.policyArn": "keyword",
+    "aws.cloudtrail.flattened.request_parameters.ipPermissions.items.ipRanges.items.cidrIp": "keyword",
+    "aws.cloudtrail.flattened.request_parameters.ipPermissions.items.fromPort": "keyword",
     "aws.cloudtrail.flattened.request_parameters.serialNumber": "keyword",
     "aws.cloudtrail.flattened.request_parameters.x-amz-server-side-encryption-customer-algorithm": "keyword",
-    "aws.cloudtrail.flattened.additional_eventdata.SSEApplied": "keyword",
-    "aws.cloudtrail.flattened.request_parameters.bucketName": "keyword",
-    "aws.cloudtrail.flattened.request_parameters.key": "keyword",
     "aws.cloudtrail.flattened.request_parameters.includeDeprecated": "keyword",
     "aws.cloudtrail.flattened.request_parameters.withDecryption": "boolean",
     "aws.cloudtrail.flattened.request_parameters.instanceId": "keyword",
-    "aws.cloudtrail.flattened.request_parameters.dryRun": "boolean",
-    "aws.cloudtrail.flattened.request_parameters.clientToken": "keyword",
-    "aws.cloudtrail.flattened.response_elements.s3BucketName": "keyword",
-    "aws.cloudtrail.flattened.response_elements.tableArn": "keyword",
     "aws.cloudtrail.flattened.request_parameters.attribute": "keyword",
     "aws.cloudtrail.flattened.request_parameters.reason": "keyword",
     "aws.cloudtrail.flattened.request_parameters.omitted": "keyword",
-    "aws.cloudtrail.flattened.request_parameters.ownersSet.items.owner": "keyword",
-    "aws.cloudtrail.flattened.response_elements.documentDescription.documentType": "keyword"
+    "aws.cloudtrail.flattened.response_elements.documentDescription.documentType": "keyword",
+    "aws.cloudtrail.flattened.request_parameters.groupSet.items.groupId": "keyword"
   },
   "logs-azure.signinlogs-*": {
     "azure.signinlogs.properties.conditional_access_audiences.application_id": "keyword",

rules/integrations/aws/credential_access_retrieve_secure_string_parameters_via_ssm.toml

@@ -2,27 +2,19 @@
 creation_date = "2024/04/12"
 integration = ["aws"]
 maturity = "production"
-updated_date = "2025/01/27"
+updated_date = "2025/07/09"
 
 [rule]
 author = ["Elastic"]
 description = """
-Detects the first occurrence of a user identity accessing AWS Systems Manager (SSM) SecureString parameters using the
-GetParameter or GetParameters API actions with credentials in the request parameters. This could indicate that the user
-is accessing sensitive information. This rule detects when a user accesses a SecureString parameter with the
-`withDecryption` parameter set to true. This is a
-[NewTerms](https://www.elastic.co/guide/en/security/current/rules-ui-create.html#create-new-terms-rule) rule that
-detects the first occurrence of a specific AWS ARN accessing SecureString parameters with decryption within the last 10
-days.
+Detects the first occurrence of a user identity accessing AWS Systems Manager (SSM) SecureString parameters using the GetParameter or GetParameters API actions with credentials in the request parameters. This could indicate that the user is accessing sensitive information. This rule detects when a user accesses a SecureString parameter with the withDecryption parameter set to true. This is a New Terms rule that detects the first occurrence of an AWS identity accessing SecureString parameters with decryption.
 """
 false_positives = [
     """
-    Users may legitimately access AWS Systems Manager (SSM) parameters using the GetParameter, GetParameters, or
-    DescribeParameters API actions with credentials in the request parameters. Ensure that the user has a legitimate
-    reason to access the parameters and that the credentials are secured.
+    Users may legitimately access AWS Systems Manager (SSM) parameters using the GetParameter, GetParameters, or DescribeParameters API actions with credentials in the request parameters. Ensure that the user has a legitimate reason to access the parameters and that the credentials are secured.
     """,
 ]
-from = "now-9m"
+from = "now-6m"
 index = ["filebeat-*", "logs-aws.cloudtrail-*"]
 language = "kuery"
 license = "Elastic License v2"
@@ -39,8 +31,8 @@ Adversaries may target SecureStrings to retrieve sensitive information such as e
 #### Possible Investigation Steps
 
 - **Review the Access Event**: Identify the specific API call (`GetParameter` or `GetParameters`) that triggered the rule. Examine the `request_parameters` for `withDecryption` set to true and the name of the accessed parameter.
-- **Verify User Identity and Access Context**: Check the `user_identity` details to understand who accessed the parameter and their role within the organization. This includes checking the ARN and access key ID to determine if the access was authorized.
-    - **User ID**: Review the `user.id` field to identify the specific user or role that initiated the API call. Note that the ARN associated may be an assumed role and may not directly correspond to a human user.
+- **Verify User Identity and Access Context**: Check the `aws.cloudtrail.user_identity` details to understand who accessed the parameter and their role within the organization. This includes checking the ARN and access key ID to determine if the access was authorized.
+    - **User ID**: Review the `user.name` field to identify the specific user or role that initiated the API call. Note that the ARN associated may be an assumed role and may not directly correspond to a human user.
 - **Contextualize with User Behavior**: Assess whether the access pattern fits the user’s normal behavior or job responsibilities. Investigate any out-of-pattern activities around the time of the event.
 - **Analyze Geographic and IP Context**: Using the `source.ip` and `source.geo` information, verify if the request came from a trusted location or if there are any anomalies that suggest a compromised account.
 - **Inspect Related CloudTrail Events**: Look for other related events in CloudTrail to see if there was unusual activity before or after this event, such as unusual login attempts, changes to permissions, or other API calls that could indicate broader unauthorized actions.
@@ -93,6 +85,22 @@ event.dataset: aws.cloudtrail
     )
 '''
 
+[rule.investigation_fields]
+field_names = [
+    "@timestamp",
+    "user.name",
+    "user_agent.original",
+    "source.ip",
+    "aws.cloudtrail.user_identity.type",
+    "aws.cloudtrail.user_identity.arn",
+    "aws.cloudtrail.user_identity.access_key_id",
+    "target.entity.id",
+    "event.action",
+    "event.outcome",
+    "cloud.account.id",
+    "cloud.region",
+    "aws.cloudtrail.request_parameters"
+]
 
 [[rule.threat]]
 framework = "MITRE ATT&CK"
@@ -114,7 +122,7 @@ reference = "https://attack.mitre.org/tactics/TA0006/"
 
 [rule.new_terms]
 field = "new_terms_fields"
-value = ["aws.cloudtrail.user_identity.arn"]
+value = ["cloud.account.id", "user.name"]
 [[rule.new_terms.history_window_start]]
 field = "history_window_start"
 value = "now-10d"

rules/integrations/aws/defense_evasion_vpc_security_group_ingress_rule_added_for_remote_connections.toml

@@ -2,26 +2,21 @@
 creation_date = "2024/04/16"
 integration = ["aws"]
 maturity = "production"
-updated_date = "2025/01/15"
+updated_date = "2025/07/10"
 
 [rule]
 author = ["Elastic"]
 description = """
-Identifies when a specified inbound (ingress) rule is added or adjusted for a VPC security group in AWS EC2. This rule
-detects when a security group rule is added that allows traffic from any IP address or from a specific IP address to
-common remote access ports, such as 22 (SSH) or 3389 (RDP). Adversaries may add these rules to allow remote access to
-VPC instances from any location, increasing the attack surface and potentially exposing the instances to unauthorized
-access.
+Identifies when a specified inbound (ingress) rule is added or adjusted for a VPC security group in AWS EC2. This rule detects when a security group rule is added that allows traffic from any IP address or from a specific IP address to common remote access ports, such as 22 (SSH) or 3389 (RDP). Adversaries may add these rules to allow remote access to VPC instances from any location, increasing the attack surface and potentially exposing the instances to unauthorized access.
 """
 false_positives = [
     """
-    Administrators may legitimately add security group rules to allow traffic from any IP address or from specific IP
-    addresses to common remote access ports.
+    Administrators may legitimately add security group rules to allow traffic from any IP address or from specific IP addresses to common remote access ports.
     """,
 ]
-from = "now-60m"
+from = "now-6m"
 index = ["filebeat-*", "logs-aws.cloudtrail-*"]
-interval = "10m"
+interval = "5m"
 language = "kuery"
 license = "Elastic License v2"
 name = "Insecure AWS EC2 VPC Security Group Ingress Rule Added"
@@ -83,11 +78,25 @@ event.dataset: "aws.cloudtrail"
     and event.provider: ec2.amazonaws.com
     and event.action: AuthorizeSecurityGroupIngress
     and event.outcome: success
-    and aws.cloudtrail.flattened.request_parameters.cidrIp: ("0.0.0.0/0" or "::/0")
-    and aws.cloudtrail.flattened.request_parameters.fromPort: (
+    and aws.cloudtrail.flattened.request_parameters.ipPermissions.items.ipRanges.items.cidrIp: ("0.0.0.0/0" or "::/0")
+    and aws.cloudtrail.flattened.request_parameters.ipPermissions.items.fromPort: (
         21 or 22 or 23 or 445 or 3389 or 5985 or 5986)
 '''
-
+[rule.investigation_fields]
+field_names = [
+    "@timestamp",
+    "user.name",
+    "user_agent.original",
+    "source.ip",
+    "aws.cloudtrail.user_identity.arn",
+    "aws.cloudtrail.user_identity.type",
+    "aws.cloudtrail.user_identity.access_key_id",
+    "event.action",
+    "event.outcome",
+    "cloud.account.id",
+    "cloud.region",
+    "aws.cloudtrail.request_parameters"
+]
 
 [[rule.threat]]
 framework = "MITRE ATT&CK"

rules/integrations/aws/discovery_ec2_deprecated_ami_discovery.toml

@@ -2,7 +2,7 @@
 creation_date = "2024/12/24"
 integration = ["aws"]
 maturity = "production"
-updated_date = "2025/06/10"
+updated_date = "2025/07/09"
 
 [rule]
 author = ["Elastic"]
@@ -39,7 +39,7 @@ This rule detects when a user queries AWS for deprecated Amazon Machine Images (
    - Analyze the `user_agent.original` field to determine the client or tool used (e.g., AWS CLI, SDK).
 
 3. **Validate the Query Context**:
-   - Inspect the `aws.cloudtrail.flattened.request_parameters` field 
+   - Inspect the `aws.cloudtrail.request_parameters` field 
    - Determine if the request is part of legitimate activity, such as:
      - Security assessments or vulnerability scans.
      - Maintenance or testing of legacy systems.
@@ -109,7 +109,6 @@ event.dataset: "aws.cloudtrail"
     and event.action: "DescribeImages"
     and event.outcome: "success"
     and aws.cloudtrail.flattened.request_parameters.includeDeprecated: "true"
-    and aws.cloudtrail.flattened.request_parameters.ownersSet.items.owner: *
 '''
 [rule.investigation_fields]
 field_names = [

rules/integrations/aws/discovery_ec2_userdata_request_for_ec2_instance.toml

@@ -2,7 +2,7 @@
 creation_date = "2024/04/14"
 integration = ["aws"]
 maturity = "production"
-updated_date = "2025/06/17"
+updated_date = "2025/07/10"
 
 [rule]
 author = ["Elastic"]
@@ -144,7 +144,7 @@ reference = "https://attack.mitre.org/tactics/TA0006/"
 
 [rule.new_terms]
 field = "new_terms_fields"
-value = ["user.name", "aws.cloudtrail.flattened.request_parameters.instanceId"]
+value = ["cloud.account.id", "user.name", "aws.cloudtrail.flattened.request_parameters.instanceId"]
 [[rule.new_terms.history_window_start]]
 field = "history_window_start"
 value = "now-14d"

rules/integrations/aws/execution_ssm_sendcommand_by_rare_user.toml

@@ -2,7 +2,7 @@
 creation_date = "2020/07/06"
 integration = ["aws"]
 maturity = "production"
-updated_date = "2025/06/25"
+updated_date = "2025/07/10"
 
 [rule]
 author = ["Elastic"]
@@ -29,8 +29,8 @@ This rule detects the execution of commands or scripts on EC2 instances using AW
 #### Possible Investigation Steps
 
 - **Identify the Target Instance**:
-  - **Instance ID**: Review the `aws.cloudtrail.flattened.request_parameters.instanceIds` field to identify which EC2 instances were targeted by this command. Confirm if these instances are expected to be managed through SSM.
-  - **Document Used**: Check the `aws.cloudtrail.flattened.request_parameters.documentName` field, which specifies the document or script being executed. Commands such as `RunShellScript` or `RunPowerShellScript` can indicate interactive sessions or script-based interactions.
+  - **Instance ID**: Review the `aws.cloudtrail.request_parameters` field to identify which EC2 instances were targeted by this command. Confirm if these instances are expected to be managed through SSM.
+  - **Document Used**: Check the `aws.cloudtrail.request_parameters` field, which specifies the name of the document or script being executed. Commands such as `RunShellScript` or `RunPowerShellScript` can indicate interactive sessions or script-based interactions.
 
 - **Review User Context**:
   - **User Identity**: Inspect the `aws.cloudtrail.user_identity.arn` field to determine the user or role executing the `SendCommand`. If this user is not typically involved in EC2 or SSM interactions, this could indicate unauthorized access.