[Rule Tuning] Updated ESQL Rules Based on Validation Results (#5151)

eric-forte-elastic · terrancedejesus · Mikaayenson · web-flow · commit 7410ec7db99a · 2025-09-30T00:36:29.000-04:00
* Updated ESQL rules based on validation results

* Patch bump

* Updated regex patterns

* added missing azure fields to non-ecs-schema.json; adjusted okta query logic to use LIKE instead of RLIKE

* fixed incorrect field in non-ecs-schema.json; changed logs-azure.signinlogs* sightings to logs-azure.signinlogs-*

* Add and

* Additional non-ecs fields

* Add EOF

* Add kibana.alert.rule.name

* removed azure.platforlogs.identity.claim.objectid; updated query for 'c07f7898-5dc3-11f0-9f27-f661ea17fbcd'

* Field removed from query removing from keep

* Patch Bump

---------

Co-authored-by: terrancedejesus &lt;terrance.dejesus@elastic.co&gt;
Co-authored-by: Mika Ayenson, PhD &lt;Mikaayenson@users.noreply.github.com&gt;
diff --git a/detection_rules/etc/integration-manifests.json.gz b/detection_rules/etc/integration-manifests.json.gz
diff --git a/detection_rules/etc/integration-schemas.json.gz b/detection_rules/etc/integration-schemas.json.gz
diff --git a/detection_rules/etc/non-ecs-schema.json b/detection_rules/etc/non-ecs-schema.json
@@ -144,7 +144,8 @@
     "signal.rule.threat.tactic.name": "keyword",
     "kibana.alert.rule.threat.tactic.id": "keyword",
     "kibana.alert.workflow_status": "keyword",
-    "kibana.alert.rule.rule_id": "keyword"
+    "kibana.alert.rule.rule_id": "keyword",
+    "kibana.alert.rule.name": "keyword"
   },
   "logs-google_workspace*": {
     "gsuite.admin": "keyword",
@@ -188,7 +189,12 @@
     "azure.auditlogs.properties.target_resources.0.display_name": "keyword",
     "azure.signinlogs.properties.authentication_details.authentication_method": "keyword",
     "azure.signinlogs.properties.authentication_processing_details": "keyword",
-    "azure.signinlogs.properties.token_protection_status_details.sign_in_session_status": "keyword"
+    "azure.signinlogs.properties.token_protection_status_details.sign_in_session_status": "keyword",
+    "azure.signinlogs.properties.session_id": "keyword",
+    "azure.signinlogs.properties.mfa_detail.auth_method": "keyword",
+    "azure.signinlogs.properties.client_credential_type": "keyword",
+    "azure.signinlogs.properties.app_owner_tenant_id": "keyword",
+    "azure.signinlogs.properties.resource_owner_tenant_id": "keyword"
   },
   "logs-azure.activitylogs-*": {
     "azure.activitylogs.properties.authentication_protocol": "keyword",
@@ -199,18 +205,22 @@
   "logs-azure.graphactivitylogs-*": {
     "azure.graphactivitylogs.properties.c_idtyp": "keyword",
     "azure.graphactivitylogs.properties.user_principal_object_id": "keyword",
-    "azure.graphactivitylogs.properties.requestUri": "keyword"
+    "azure.graphactivitylogs.properties.requestUri": "keyword",
+    "azure.graphactivitylogs.properties.c_sid": "keyword"
   },
   "logs-azure.auditlogs-*": {
     "azure.auditlogs.properties.target_resources.0.modified_properties.1.display_name": "keyword",
     "azure.auditlogs.properties.target_resources.0.modified_properties.1.new_value": "keyword",
     "azure.auditlogs.properties.target_resources.0.modified_properties.3.new_value": "keyword",
     "azure.auditlogs.properties.target_resources.0.modified_properties.2.new_value": "keyword",
-    "azure.auditlogs.properties.additional_details.value": "keyword"
+    "azure.auditlogs.properties.additional_details.value": "keyword",
+    "azure.auditlogs.properties.target_resources.0.modified_properties.0.new_value": "keyword",
+    "azure.auditlogs.properties.target_resources.0.modified_properties.0.old_value": "keyword"
   },
   "logs-azure.platformlogs-*": {
     "azure.platformlogs.identity.claim.upn": "keyword",
-    "azure.platformlogs.properties.id": "keyword"
+    "azure.platformlogs.properties.id": "keyword",
+    "azure.platformlogs.identity.claim.appid": "keyword"
   },
    "logs-o365.audit-*": {
      "o365.audit.ExtendedProperties.RequestType": "keyword",
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "detection_rules"
-version = "1.4.7"
+version = "1.4.8"
 description = "Detection Rules is the home for rules used by Elastic Security. This repository is used for the development, maintenance, testing, validation, and release of rules for Elastic Security’s Detection Engine."
 readme = "README.md"
 requires-python = ">=3.12"
diff --git a/rules/integrations/aws/impact_aws_s3_bucket_enumeration_or_brute_force.toml b/rules/integrations/aws/impact_aws_s3_bucket_enumeration_or_brute_force.toml
@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2024/05/01"
 maturity = "production"
-updated_date = "2025/07/16"
+updated_date = "2025/09/25"
 
 [rule]
 author = ["Elastic"]
@@ -85,10 +85,11 @@ timestamp_override = "event.ingested"
 type = "esql"
 
 query = '''
-from logs-aws.cloudtrail*
+from logs-aws.cloudtrail-*
 
 | where
-  event.provider == "s3.amazonaws.com"
+  event.dataset == "aws.cloudtrail"
+  and event.provider == "s3.amazonaws.com"
   and aws.cloudtrail.error_code == "AccessDenied"
   and tls.client.server_name is not null
   and cloud.account.id is not null
diff --git a/rules/integrations/aws/impact_s3_static_site_js_file_uploaded.toml b/rules/integrations/aws/impact_s3_static_site_js_file_uploaded.toml
@@ -2,7 +2,7 @@
 creation_date = "2025/04/15"
 integration = ["aws"]
 maturity = "production"
-updated_date = "2025/07/16"
+updated_date = "2025/09/25"
 
 [rule]
 author = ["Elastic"]
@@ -96,7 +96,7 @@ from logs-aws.cloudtrail* metadata _id, _version, _index
     "%{{?bucket.name.key}=%{Esql.aws_cloudtrail_request_parameters_bucket_name}, %{?host.key}=%{Esql_priv.aws_cloudtrail_request_parameters_host}, %{?bucket.object.location.key}=%{Esql.aws_cloudtrail_request_parameters_bucket_object_location}}"
 
 // Extract file name portion from full object path
-| dissect Esql.aws_cloudtrail_request_parameters_object_location "%{}static/js/%{Esql.aws_cloudtrail_request_parameters_object_key}"
+| dissect Esql.aws_cloudtrail_request_parameters_bucket_object_location "%{}static/js/%{Esql.aws_cloudtrail_request_parameters_object_key}"
 
 // Match on JavaScript files
 | where ends_with(Esql.aws_cloudtrail_request_parameters_object_key, ".js")
diff --git a/rules/integrations/aws_bedrock/aws_bedrock_execution_without_guardrails.toml b/rules/integrations/aws_bedrock/aws_bedrock_execution_without_guardrails.toml
@@ -1,7 +1,8 @@
 [metadata]
 creation_date = "2024/11/25"
+integration = ["aws_bedrock"]
 maturity = "production"
-updated_date = "2025/07/16"
+updated_date = "2025/09/25"
 
 [rule]
 author = ["Elastic"]
diff --git a/rules/integrations/aws_bedrock/aws_bedrock_guardrails_multiple_violations_by_single_user.toml b/rules/integrations/aws_bedrock/aws_bedrock_guardrails_multiple_violations_by_single_user.toml
@@ -1,7 +1,8 @@
 [metadata]
 creation_date = "2024/05/02"
+integration = ["aws_bedrock"]
 maturity = "production"
-updated_date = "2025/07/16"
+updated_date = "2025/09/25"
 
 [rule]
 author = ["Elastic"]
diff --git a/rules/integrations/aws_bedrock/aws_bedrock_guardrails_multiple_violations_in_single_request.toml b/rules/integrations/aws_bedrock/aws_bedrock_guardrails_multiple_violations_in_single_request.toml
@@ -1,7 +1,8 @@
 [metadata]
 creation_date = "2024/05/02"
+integration = ["aws_bedrock"]
 maturity = "production"
-updated_date = "2025/07/16"
+updated_date = "2025/09/25"
 
 [rule]
 author = ["Elastic"]
diff --git a/rules/integrations/aws_bedrock/aws_bedrock_high_confidence_misconduct_blocks_detected.toml b/rules/integrations/aws_bedrock/aws_bedrock_high_confidence_misconduct_blocks_detected.toml
@@ -1,7 +1,8 @@
 [metadata]
 creation_date = "2024/05/05"
+integration = ["aws_bedrock"]
 maturity = "production"
-updated_date = "2025/07/16"
+updated_date = "2025/09/25"
 
 [rule]
 author = ["Elastic"]
diff --git a/rules/integrations/aws_bedrock/aws_bedrock_high_resource_consumption_detection.toml b/rules/integrations/aws_bedrock/aws_bedrock_high_resource_consumption_detection.toml
@@ -1,7 +1,8 @@
 [metadata]
 creation_date = "2024/05/04"
+integration = ["aws_bedrock"]
 maturity = "production"
-updated_date = "2025/07/16"
+updated_date = "2025/09/25"
 
 [rule]
 author = ["Elastic"]
diff --git a/rules/integrations/aws_bedrock/aws_bedrock_multiple_attempts_to_use_denied_models_by_user.toml b/rules/integrations/aws_bedrock/aws_bedrock_multiple_attempts_to_use_denied_models_by_user.toml
@@ -1,7 +1,8 @@
 [metadata]
 creation_date = "2024/05/02"
+integration = ["aws_bedrock"]
 maturity = "production"
-updated_date = "2025/07/16"
+updated_date = "2025/09/25"
 
 [rule]
 author = ["Elastic"]
diff --git a/rules/integrations/aws_bedrock/aws_bedrock_multiple_sensitive_information_policy_blocks_detected.toml b/rules/integrations/aws_bedrock/aws_bedrock_multiple_sensitive_information_policy_blocks_detected.toml
@@ -1,7 +1,8 @@
 [metadata]
 creation_date = "2024/11/20"
+integration = ["aws_bedrock"]
 maturity = "production"
-updated_date = "2025/07/16"
+updated_date = "2025/09/25"
 
 [rule]
 author = ["Elastic"]
diff --git a/rules/integrations/aws_bedrock/aws_bedrock_multiple_topic_policy_blocks_detected.toml b/rules/integrations/aws_bedrock/aws_bedrock_multiple_topic_policy_blocks_detected.toml
@@ -1,7 +1,8 @@
 [metadata]
 creation_date = "2024/11/20"
+integration = ["aws_bedrock"]
 maturity = "production"
-updated_date = "2025/07/16"
+updated_date = "2025/09/25"
 
 [rule]
 author = ["Elastic"]
diff --git a/rules/integrations/aws_bedrock/aws_bedrock_multiple_word_policy_blocks_detected.toml b/rules/integrations/aws_bedrock/aws_bedrock_multiple_word_policy_blocks_detected.toml
@@ -1,7 +1,8 @@
 [metadata]
 creation_date = "2024/11/20"
+integration = ["aws_bedrock"]
 maturity = "production"
-updated_date = "2025/07/16"
+updated_date = "2025/09/25"
 
 [rule]
 author = ["Elastic"]
diff --git a/rules/integrations/azure/credential_access_azure_entra_suspicious_signin.toml b/rules/integrations/azure/credential_access_azure_entra_suspicious_signin.toml
@@ -2,7 +2,7 @@
 creation_date = "2025/04/28"
 integration = ["azure"]
 maturity = "production"
-updated_date = "2025/07/16"
+updated_date = "2025/09/26"
 
 [rule]
 author = ["Elastic"]
@@ -69,7 +69,7 @@ timestamp_override = "event.ingested"
 type = "esql"
 
 query = '''
-from logs-azure.signinlogs* metadata _id, _version, _index
+from logs-azure.signinlogs-* metadata _id, _version, _index
 
 // Scheduled to run every hour, reviewing events from past hour
 | where
diff --git a/rules/integrations/azure/credential_access_azure_entra_totp_brute_force_attempts.toml b/rules/integrations/azure/credential_access_azure_entra_totp_brute_force_attempts.toml
@@ -2,7 +2,7 @@
 creation_date = "2024/12/11"
 integration = ["azure"]
 maturity = "production"
-updated_date = "2025/07/31"
+updated_date = "2025/09/26"
 
 [rule]
 author = ["Elastic"]
@@ -85,7 +85,7 @@ timestamp_override = "event.ingested"
 type = "esql"
 
 query = '''
-from logs-azure.signinlogs* metadata _id, _version, _index
+from logs-azure.signinlogs-* metadata _id, _version, _index
 
 | where
     // filter for Entra Sign-in Logs
diff --git a/rules/integrations/azure/credential_access_azure_key_vault_excessive_retrieval.toml b/rules/integrations/azure/credential_access_azure_key_vault_excessive_retrieval.toml
@@ -2,7 +2,7 @@
 creation_date = "2025/07/10"
 integration = ["azure"]
 maturity = "production"
-updated_date = "2025/07/24"
+updated_date = "2025/09/26"
 
 [rule]
 author = ["Elastic"]
@@ -121,7 +121,6 @@ from logs-azure.platformlogs-* metadata _id, _index
     Esql_priv.azure_platformlogs_identity_claim_upn_values = values(azure.platformlogs.identity.claim.upn),
     Esql.azure_platformlogs_identity_claim_upn_count_distinct = count_distinct(azure.platformlogs.identity.claim.upn),
     Esql.azure_platformlogs_identity_claim_appid_values = values(azure.platformlogs.identity.claim.appid),
-    Esql.azure_platformlogs_identity_claim_objectid_values = values(azure.platformlogs.identity.claim.objectid),
 
     Esql.source_ip_values = values(source.ip),
     Esql.geo_city_values = values(geo.city_name),
@@ -150,7 +149,6 @@ by Esql.time_window_date_trunc, azure.platformlogs.identity.claim.upn
     Esql_priv.azure_platformlogs_identity_claim_upn_values,
     Esql.azure_platformlogs_identity_claim_upn_count_distinct,
     Esql.azure_platformlogs_identity_claim_appid_values,
-    Esql.azure_platformlogs_identity_claim_objectid_values,
     Esql.source_ip_values,
     Esql.geo_city_values,
     Esql.geo_region_values,
diff --git a/rules/integrations/azure/credential_access_entra_id_brute_force_activity.toml b/rules/integrations/azure/credential_access_entra_id_brute_force_activity.toml
@@ -2,7 +2,7 @@
 creation_date = "2024/09/06"
 integration = ["azure"]
 maturity = "production"
-updated_date = "2025/09/10"
+updated_date = "2025/09/26"
 
 [rule]
 author = ["Elastic"]
@@ -88,7 +88,7 @@ timestamp_override = "event.ingested"
 type = "esql"
 
 query = '''
-from logs-azure.signinlogs*
+from logs-azure.signinlogs-*
 
 // Define a time window for grouping and maintain the original event timestamp
 | eval Esql.time_window_date_trunc = date_trunc(15 minutes, @timestamp)
diff --git a/rules/integrations/azure/credential_access_entra_id_excessive_account_lockouts.toml b/rules/integrations/azure/credential_access_entra_id_excessive_account_lockouts.toml
@@ -2,7 +2,7 @@
 creation_date = "2025/07/01"
 integration = ["azure"]
 maturity = "production"
-updated_date = "2025/07/16"
+updated_date = "2025/09/26"
 
 [rule]
 author = ["Elastic"]
@@ -84,7 +84,7 @@ timestamp_override = "event.ingested"
 type = "esql"
 
 query = '''
-from logs-azure.signinlogs*
+from logs-azure.signinlogs-*
 
 | eval
     Esql.time_window_date_trunc = date_trunc(30 minutes, @timestamp),
diff --git a/rules/integrations/azure/credential_access_entra_signin_brute_force_microsoft_365.toml b/rules/integrations/azure/credential_access_entra_signin_brute_force_microsoft_365.toml
@@ -2,7 +2,7 @@
 creation_date = "2024/09/06"
 integration = ["azure"]
 maturity = "production"
-updated_date = "2025/07/16"
+updated_date = "2025/09/26"
 
 [rule]
 author = ["Elastic"]
@@ -88,7 +88,7 @@ timestamp_override = "event.ingested"
 type = "esql"
 
 query = '''
-from logs-azure.signinlogs*
+from logs-azure.signinlogs-*
 
 | eval
     Esql.time_window_date_trunc = date_trunc(15 minutes, @timestamp),
diff --git a/rules/integrations/azure/initial_access_entra_id_high_risk_signin.toml b/rules/integrations/azure/initial_access_entra_id_high_risk_signin.toml
@@ -2,7 +2,7 @@
 creation_date = "2021/01/04"
 integration = ["azure"]
 maturity = "production"
-updated_date = "2025/05/21"
+updated_date = "2025/09/26"
 
 [rule]
 author = ["Elastic", "Willem D'Haese"]
@@ -13,7 +13,7 @@ provide specific details about how risk is calculated, each level brings higher
 compromised.
 """
 from = "now-9m"
-index = ["filebeat-*", "logs-azure.signinlogs*"]
+index = ["filebeat-*", "logs-azure.signinlogs-*"]
 language = "kuery"
 license = "Elastic License v2"
 name = "Microsoft Entra ID High Risk Sign-in"
diff --git a/rules/integrations/azure/initial_access_entra_id_suspicious_oauth_flow_via_auth_broker_to_drs.toml b/rules/integrations/azure/initial_access_entra_id_suspicious_oauth_flow_via_auth_broker_to_drs.toml
@@ -2,7 +2,7 @@
 creation_date = "2025/04/30"
 integration = ["azure"]
 maturity = "production"
-updated_date = "2025/07/16"
+updated_date = "2025/09/26"
 
 [rule]
 author = ["Elastic"]
@@ -89,7 +89,7 @@ timestamp_override = "event.ingested"
 type = "esql"
 
 query = '''
-from logs-azure.signinlogs* metadata _id, _version, _index
+from logs-azure.signinlogs-* metadata _id, _version, _index
 | where
     event.dataset == "azure.signinlogs" and
     event.outcome == "success" and
diff --git a/rules/integrations/azure_openai/azure_openai_denial_of_ml_service_detection.toml b/rules/integrations/azure_openai/azure_openai_denial_of_ml_service_detection.toml
@@ -1,7 +1,8 @@
 [metadata]
 creation_date = "2025/02/25"
+integration = ["azure_openai"]
 maturity = "production"
-updated_date = "2025/07/16"
+updated_date = "2025/09/25"
 
 [rule]
 author = ["Elastic"]
diff --git a/rules/integrations/azure_openai/azure_openai_insecure_output_handling_detection.toml b/rules/integrations/azure_openai/azure_openai_insecure_output_handling_detection.toml
@@ -1,7 +1,8 @@
 [metadata]
 creation_date = "2025/02/25"
+integration = ["azure_openai"]
 maturity = "production"
-updated_date = "2025/07/16"
+updated_date = "2025/09/25"
 
 [rule]
 author = ["Elastic"]
diff --git a/rules/integrations/azure_openai/azure_openai_model_theft_detection.toml b/rules/integrations/azure_openai/azure_openai_model_theft_detection.toml
@@ -1,7 +1,8 @@
 [metadata]
 creation_date = "2025/02/25"
+integration = ["azure_openai"]
 maturity = "production"
-updated_date = "2025/07/16"
+updated_date = "2025/09/25"
 
 [rule]
 author = ["Elastic"]
diff --git a/rules/integrations/okta/credential_access_multiple_device_token_hashes_for_single_okta_session.toml b/rules/integrations/okta/credential_access_multiple_device_token_hashes_for_single_okta_session.toml
@@ -2,7 +2,7 @@
 creation_date = "2023/11/08"
 integration = ["okta"]
 maturity = "production"
-updated_date = "2025/07/16"
+updated_date = "2025/09/25"
 
 [rule]
 author = ["Elastic"]
@@ -86,7 +86,7 @@ from logs-okta*
         "user.authentication.sso"
     ) and
     okta.actor.alternate_id != "system@okta.com" and
-    okta.actor.alternate_id rlike "[^@\s]+\@[^@\s]+" and
+    okta.actor.alternate_id rlike "[^@\\s]+\\@[^@\\s]+" and
     okta.authentication_context.external_session_id != "unknown"
 | keep
     event.action,
diff --git a/rules/integrations/okta/credential_access_okta_authentication_for_multiple_users_from_single_source.toml b/rules/integrations/okta/credential_access_okta_authentication_for_multiple_users_from_single_source.toml
diff --git a/rules/integrations/okta/credential_access_okta_authentication_for_multiple_users_with_the_same_device_token_hash.toml b/rules/integrations/okta/credential_access_okta_authentication_for_multiple_users_with_the_same_device_token_hash.toml
diff --git a/rules/integrations/okta/credential_access_okta_multiple_device_token_hashes_for_single_user.toml b/rules/integrations/okta/credential_access_okta_multiple_device_token_hashes_for_single_user.toml
diff --git a/rules/integrations/okta/initial_access_okta_user_sessions_started_from_different_geolocations.toml b/rules/integrations/okta/initial_access_okta_user_sessions_started_from_different_geolocations.toml
