Merge branch 'esql-field-validation' of https://github.com/elastic/detection-rules into esql-field-validation

Author: eric-forte-elastic

detection_rules/rule.py

@@ -1006,10 +1006,14 @@ class ThreatMatchRuleData(QueryRuleData):
     @dataclass(frozen=True)
     class Entries:
         @dataclass(frozen=True)
-        class ThreatMapEntry:
+        class ThreatMapEntry(StackCompatMixin):
             field: definitions.NonEmptyStr
             type: Literal["mapping"]
             value: definitions.NonEmptyStr
+            # Use dataclasses.field to avoid shadowing by attribute name "field"
+            negate: bool | None = dataclasses.field(  # type: ignore[reportIncompatibleVariableOverride]
+                metadata={"metadata": {"min_compat": "9.2"}}
+            )
 
         entries: list[ThreatMapEntry]
 
@@ -1042,6 +1046,38 @@ def validate_query(self, meta: RuleMeta) -> None:
 
             threat_query_validator.validate(self, meta)
 
+    def validate(self, meta: RuleMeta) -> None:  # noqa: ARG002
+        """Validate negate usage and group semantics for threat mapping."""
+
+        for idx, group in enumerate(self.threat_mapping or []):
+            entries = group.entries or []
+
+            # Enforce: DOES NOT MATCH entries are allowed only if there is at least
+            # one MATCH (non-negated) entry in the same group
+            has_negate = any(bool(getattr(e, "negate", False)) for e in entries)
+            has_match = any(not bool(getattr(e, "negate", False)) for e in entries)
+            if has_negate and not has_match:
+                msg = (
+                    f"threat_mapping group {idx}: DOES NOT MATCH entries require at least one MATCH "
+                    "(non-negated) entry in the same group."
+                )
+                raise ValidationError(msg)
+
+            # Track negate presence per (source.field, indicator.field) pair to detect
+            # conflicts where both MATCH and DOES NOT MATCH are defined for the same pair
+            pair_to_negates: dict[tuple[str, str], set[bool]] = {}
+            for e in entries:
+                is_neg = bool(getattr(e, "negate", False))
+                pair_to_negates.setdefault((e.field, e.value), set()).add(is_neg)
+
+            for (src_field, ind_field), flags in pair_to_negates.items():
+                if True in flags and False in flags:
+                    msg = (
+                        f"threat_mapping group {idx}: cannot define both MATCH and DOES NOT MATCH for the same "
+                        f"source and indicator fields: '{src_field}' <-> '{ind_field}'."
+                    )
+                    raise ValidationError(msg)
+
 
 # All of the possible rule types
 # Sort inverse of any inheritance - see comment in TOMLRuleContents.to_dict

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "detection_rules"
-version = "1.4.0"
+version = "1.5.0"
 description = "Detection Rules is the home for rules used by Elastic Security. This repository is used for the development, maintenance, testing, validation, and release of rules for Elastic Security’s Detection Engine."
 readme = "README.md"
 requires-python = ">=3.12"

rules/linux/persistence_dbus_service_creation.toml

@@ -2,7 +2,7 @@
 creation_date = "2025/01/16"
 integration = ["endpoint", "sentinel_one_cloud_funnel"]
 maturity = "production"
-updated_date = "2025/03/20"
+updated_date = "2025/09/09"
 
 [rule]
 author = ["Elastic"]
@@ -117,7 +117,6 @@ file.extension in ("service", "conf") and file.path like~ (
     "/usr/sbin/sshd", "/usr/bin/gitlab-runner", "/opt/gitlab/embedded/bin/ruby", "/usr/sbin/gdm", "/usr/bin/install",
     "/usr/local/manageengine/uems_agent/bin/dcregister"
   ) or
-  file.Ext.original.extension == "dpkg-new" or
   process.executable : (
     "/nix/store/*", "/var/lib/dpkg/*", "/tmp/vmis.*", "/snap/*", "/dev/fd/*", "/usr/lib/virtualbox/*"
   ) or

rules/windows/command_and_control_common_llm_endpoint.toml

@@ -2,7 +2,7 @@
 creation_date = "2025/09/01"
 integration = ["endpoint", "windows", "sentinel_one_cloud_funnel"]
 maturity = "production"
-updated_date = "2025/09/01"
+updated_date = "2025/09/05"
 
 
 [rule]
@@ -78,7 +78,10 @@ network where host.os.type == "windows" and dns.question.name != null and
 
   ?process.code_signature.subject_name : ("AutoIt Consulting Ltd", "OpenJS Foundation", "Python Software Foundation") or
 
-  (process.executable : ("?:\\Users\\*.exe", "", "?:\\ProgramData\\*.exe") and ?process.code_signature.trusted != true)
+  (
+    process.executable : ("?:\\Users\\*.exe", "?:\\ProgramData\\*.exe") and
+    (?process.code_signature.trusted == false or ?process.code_signature.exists == false)
+  )
  ) and
     dns.question.name : (
     // Major LLM APIs

rules/windows/command_and_control_dns_susp_tld.toml

@@ -2,7 +2,7 @@
 creation_date = "2025/08/20"
 integration = ["endpoint", "windows", "sentinel_one_cloud_funnel", "crowdstrike"]
 maturity = "production"
-updated_date = "2025/08/20"
+updated_date = "2025/09/05"
 
 [rule]
 author = ["Elastic"]
@@ -77,9 +77,9 @@ network where host.os.type == "windows" and dns.question.name != null and
   process.name : ("MSBuild.exe", "mshta.exe", "wscript.exe", "powershell.exe", "pwsh.exe", "msiexec.exe", "rundll32.exe",
                   "bitsadmin.exe", "InstallUtil.exe", "python.exe", "regsvr32.exe", "dllhost.exe", "node.exe",
                   "java.exe", "javaw.exe", "*.pif", "*.com", "*.scr") or
-  ?process.code_signature.trusted != true or
+  (?process.code_signature.trusted == false or ?process.code_signature.exists == false) or
   ?process.code_signature.subject_name : ("AutoIt Consulting Ltd", "OpenJS Foundation", "Python Software Foundation") or
-  process.executable : ("?:\\Users\\*.exe", "", "?:\\ProgramData\\*.exe")
+  process.executable : ("?:\\Users\\*.exe", "?:\\ProgramData\\*.exe")
  ) and
 dns.question.name regex """.*\.(top|buzz|xyz|rest|ml|cf|gq|ga|onion|monster|cyou|quest|cc|bar|cfd|click|cam|surf|tk|shop|club|icu|pw|ws|online|fun|life|boats|store|hair|skin|motorcycles|christmas|lol|makeup|mom|bond|beauty|biz|live|work|zip|country|accountant|date|party|science|loan|win|men|faith|review|racing|download|host)"""
 '''

rules/windows/command_and_control_remote_file_copy_powershell.toml

@@ -2,7 +2,7 @@
 creation_date = "2020/11/30"
 integration = ["endpoint"]
 maturity = "production"
-updated_date = "2025/02/03"
+updated_date = "2025/09/04"
 
 [transform]
 [[transform.osquery]]
@@ -133,18 +133,29 @@ type = "eql"
 
 query = '''
 sequence by process.entity_id with maxspan=30s
-
-[network where host.os.type == "windows" and
-  process.name : ("powershell.exe", "pwsh.exe", "powershell_ise.exe") and network.protocol == "dns" and
-   not dns.question.name : (
-          "localhost", "*.microsoft.com", "*.azureedge.net", "*.powershellgallery.com",
-          "*.windowsupdate.com", "metadata.google.internal", "dist.nuget.org",
-          "artifacts.elastic.co", "*.digicert.com", "packages.chocolatey.org",
-          "outlook.office365.com"
-       ) and not user.id : "S-1-5-18"]
+[network where host.os.type == "windows" and 
+  process.name : ("powershell.exe", "pwsh.exe", "powershell_ise.exe") and
+  network.protocol == "dns" and
+  not dns.question.name : (
+        "*.microsoft.com", "*.azureedge.net", "*.powershellgallery.com", "*.windowsupdate.com",
+        "metadata.google.internal", "dist.nuget.org", "artifacts.elastic.co", "*.digicert.com",
+        "*.chocolatey.org", "outlook.office365.com", "cdn.oneget.org", "ci.dot.net",
+        "packages.icinga.com", "login.microsoftonline.com", "*.gov", "*.azure.com", "*.python.org",
+        "dl.google.com", "sensor.cloud.tenable.com", "*.azurefd.net", "*.office.net", "*.anac*",
+        "aka.ms", "dot.net", "*.visualstudio.com", "*.local") and
+  not user.id == "S-1-5-18" and
+  /* Filter out NetBIOS/LLMNR-style names (e.g. host, localhost, etc.) */
+  dns.question.name regex """.*\.[a-zA-Z]{2,5}"""]
 [file where host.os.type == "windows" and event.type == "creation" and
-  process.name : "powershell.exe" and file.extension : ("exe", "dll", "ps1", "bat") and
-  not file.name : "__PSScriptPolicy*.ps1"]
+  process.name : "powershell.exe" and 
+  (file.extension : ("exe", "dll", "ps1", "bat") or file.Ext.header_bytes : "4d5a*") and
+  not file.name : "__PSScriptPolicy*.ps1" and
+  not file.path : (
+        "?:\\Users\\*\\AppData\\Local\\Temp\\????????.dll",
+        "?:\\Users\\*\\AppData\\Local\\Temp\\*\\????????.dll",
+        "?:\\Windows\\TEMP\\ansible-tmp-*\\AnsiballZ*.ps1"
+  ) and
+  not user.id == "S-1-5-18"]
 '''
 
 

rules/windows/defense_evasion_untrusted_driver_loaded.toml

@@ -2,7 +2,7 @@
 creation_date = "2023/01/27"
 integration = ["endpoint"]
 maturity = "production"
-updated_date = "2025/02/03"
+updated_date = "2025/09/04"
 
 [transform]
 [[transform.osquery]]
@@ -112,7 +112,7 @@ type = "eql"
 
 query = '''
 driver where host.os.type == "windows" and process.pid == 4 and
-  dll.code_signature.trusted != true and 
+  (dll.code_signature.trusted == false or dll.code_signature.exists == false) and
   not dll.code_signature.status : ("errorExpired", "errorRevoked", "errorCode_endpoint:*")
 '''
 

rules/windows/discovery_host_public_ip_address_lookup.toml

@@ -2,7 +2,7 @@
 creation_date = "2025/08/20"
 integration = ["endpoint", "windows", "sentinel_one_cloud_funnel", "crowdstrike"]
 maturity = "production"
-updated_date = "2025/08/20"
+updated_date = "2025/09/05"
 
 [rule]
 author = ["Elastic"]
@@ -78,11 +78,11 @@ network where host.os.type == "windows" and dns.question.name != null and
   "bitsadmin.exe", "InstallUtil.exe", "RegAsm.exe", "vbc.exe", "RegSvcs.exe", "python.exe", "regsvr32.exe", "dllhost.exe",
   "node.exe", "javaw.exe", "java.exe", "*.pif", "*.com") or
 
-  ?process.code_signature.trusted != true or
+  (?process.code_signature.trusted == false or ?process.code_signature.exists == false) or
 
   ?process.code_signature.subject_name : ("AutoIt Consulting Ltd", "OpenJS Foundation", "Python Software Foundation") or
 
-  ?process.executable : ("?:\\Users\\*.exe", "", "?:\\ProgramData\\*.exe",  "?\\Device\\HarddiskVolume?\\Users\\*.exe",  "?\\Device\\HarddiskVolume?\\ProgramData\\*.exe")
+  ?process.executable : ("?:\\Users\\*.exe", "?:\\ProgramData\\*.exe", "?\\Device\\HarddiskVolume?\\Users\\*.exe", "?\\Device\\HarddiskVolume?\\ProgramData\\*.exe")
  ) and
  dns.question.name :
          (