Merge branch 'main' into esql-field-validation

Author: eric-forte-elastic

detection_rules/etc/custom-consolidated-rules.ndjson

@@ -43,7 +43,7 @@
         "TargetLogonId": "keyword",
         "TargetProcessGUID": "keyword",
         "TargetSid": "keyword",
-      	"SchemaFriendlyName": "keyword",
+        "SchemaFriendlyName": "keyword",
         "Resource": "keyword",
         "RpcCallClientLocality": "keyword",
         "PrivilegeList": "keyword",
@@ -207,5 +207,17 @@
   "logs-okta*": {
     "okta.debug_context.debug_data.flattened.requestedScopes": "keyword",
     "okta.debug_context.debug_data.flattened.grantType": "keyword"
+  },
+  "logs-network_traffic.http*": {
+    "data_stream.dataset": "keyword",
+    "url.path": "keyword",
+    "http.request.referrer": "keyword",
+    "http.request.headers.content-type": "keyword",
+    "network.direction": "keyword",
+    "http.request.method": "keyword",
+    "request": "keyword",
+    "http.request.body.bytes": "long",
+    "http.request.body.content": "keyword",
+    "http.response.headers.server": "keyword"
   }
 }

detection_rules/etc/non-ecs-schema.json

@@ -12,20 +12,23 @@ echo "Performing a quick rule alerts search..."
 echo "Requires .detection-rules-cfg.json credentials file set."
 python -m detection_rules kibana search-alerts
 
-echo "Performing a rule export..."
-mkdir tmp-export 2>/dev/null
-python -m detection_rules kibana export-rules -d tmp-export -sv --skip-errors -r 565d6ca5-75ba-4c82-9b13-add25353471c
-ls tmp-export
-echo "Removing generated files..."
-rm -rf tmp-export
+echo "Setting Up Custom Directory..."
+mkdir tmp-custom 2>/dev/null
+python -m detection_rules custom-rules setup-config tmp-custom
+export CUSTOM_RULES_DIR=./tmp-custom/
 
-echo "Performing a rule import..."
+echo "Performing a rule conversion from ndjson to toml files..."
+python -m detection_rules import-rules-to-repo detection_rules/etc/custom-consolidated-rules.ndjson -ac -e -s $CUSTOM_RULES_DIR/rules --required-only
+
+echo "Performing a rule import to kibana..."
 
-python -m detection_rules custom-rules setup-config tmp-custom
-export CUSTOM_RULES_DIR=./tmp-custom
-cp rules/threat_intel/threat_intel_indicator_match_address.toml tmp-custom/rules/
 python -m detection_rules kibana import-rules -o -e -ac
-rm -rf tmp-custom
+
+echo "Performing a rule export..."
+python -m detection_rules kibana export-rules -d $CUSTOM_RULES_DIR -ac -e -sv --custom-rules-only 
+
+echo "Removing generated files..."
+rm -rf $CUSTOM_RULES_DIR
 set -e CUSTOM_RULES_DIR
 
 echo "Detection-rules Remote CLI tests completed!"

detection_rules/etc/test_remote_cli.bash

@@ -48,6 +48,14 @@
       ]
     }
   },
+  {
+    "metadata": {
+      "field": "value"
+    },
+    "rule": {
+      "path": "?:\\\\Windows\\\\Sys?????\\\\x5lrs.dll"
+    }
+  },
   {
     "metadata": {
       "field": "value"

detection_rules/etc/test_toml.json

@@ -510,10 +510,10 @@
     "version": 4
   },
   "0d3d2254-2b4a-11f0-a019-f661ea17fbcc": {
-    "rule_name": "Microsoft Entra ID Session Reuse with Suspicious Graph Access",
-    "sha256": "2ff9a11a69b39d114739b56e1264c1c56b7fa7879955c39fc95314719ddfd722",
+    "rule_name": "Microsoft Entra ID Suspicious Session Reuse to Graph Access",
+    "sha256": "5d51cd77e355a15effce25681d7c34951a0d647ed54067f8a00cecb2d06c3894",
     "type": "esql",
-    "version": 3
+    "version": 4
   },
   "0d69150b-96f8-467c-a86d-a67a3378ce77": {
     "rule_name": "Nping Process Activity",
@@ -1303,9 +1303,9 @@
   },
   "203ab79b-239b-4aa5-8e54-fc50623ee8e4": {
     "rule_name": "Creation or Modification of Root Certificate",
-    "sha256": "a029643dc698af540c0359ee8ad1f382db3e999941b3514b9d07b2561ee7140c",
+    "sha256": "cb97ac512379616b3ee47f87a9d7a7f6cdc27f77c1aeb2207f6fa1bbc5fa06af",
     "type": "eql",
-    "version": 313
+    "version": 314
   },
   "2045567e-b0af-444a-8c0b-0b6e2dae9e13": {
     "rule_name": "AWS Route 53 Domain Transferred to Another Account",
@@ -1937,6 +1937,12 @@
     "type": "query",
     "version": 107
   },
+  "32144184-7bfa-4541-9c3f-b65f16d24df9": {
+    "rule_name": "Potential Web Shell ASPX File Creation",
+    "sha256": "706d6f81cd64e9b7c43d7e6547570fcd8295082645940422412c06cc142acb03",
+    "type": "eql",
+    "version": 1
+  },
   "3216949c-9300-4c53-b57a-221e364c6457": {
     "rule_name": "Unusual High Word Policy Blocks Detected",
     "sha256": "5e62d95bdfadfdce8505ea429f74acce99d2c32d8fc2ca48883884f599022754",
@@ -2485,9 +2491,9 @@
   },
   "403ef0d3-8259-40c9-a5b6-d48354712e49": {
     "rule_name": "Unusual Persistence via Services Registry",
-    "sha256": "953108f9385058fa30661eb24193e480e26db93fe546bc034e3e0844a84afe66",
+    "sha256": "3b86134e6a85714e4676aa01b2952e1a4936c55d61269d6858ab4364c23badd8",
     "type": "eql",
-    "version": 313
+    "version": 314
   },
   "40ddbcc8-6561-44d9-afc8-eefdbfe0cccd": {
     "rule_name": "Suspicious Modprobe File Event",
@@ -2851,9 +2857,9 @@
   },
   "4bd1c1af-79d4-4d37-9efa-6e0240640242": {
     "rule_name": "Unusual Process Execution Path - Alternate Data Stream",
-    "sha256": "631a873fb859163e59464b6b025f23707878dd21c31102ac27a712cbacec2dfe",
+    "sha256": "08f92365c8289d32623711be239952da8e2d840c26fc0c8cd00126ee17684e8f",
     "type": "eql",
-    "version": 313
+    "version": 314
   },
   "4c3c6c47-e38f-4944-be27-5c80be973bd7": {
     "rule_name": "Unusual SSHD Child Process",
@@ -3037,9 +3043,9 @@
   },
   "52aaab7b-b51c-441a-89ce-4387b3aea886": {
     "rule_name": "Unusual Network Connection via RunDLL32",
-    "sha256": "ae3612661681845eb5f46b07712020784c7c2dd342d10442378a84ae63049b17",
+    "sha256": "9a11f66a5f52ddf8e32658df86dc2ad920a342a4f635228e92331ddee8942239",
     "type": "eql",
-    "version": 211
+    "version": 212
   },
   "52afbdc5-db15-485e-bc24-f5707f820c4b": {
     "rule_name": "Unusual Linux Network Activity",
@@ -3323,6 +3329,12 @@
     "type": "eql",
     "version": 12
   },
+  "5a876e0d-d39a-49b9-8ad8-19c9b622203b": {
+    "rule_name": "Command Line Obfuscation via Whitespace Padding",
+    "sha256": "e8e4200bfd160124ebd18fa2e0136a6e6a467bbd77c38003b4679d2c28ac425a",
+    "type": "esql",
+    "version": 1
+  },
   "5ab49127-b1b3-46e6-8a38-9e8512a2a363": {
     "rule_name": "ROT Encoded Python Script Execution",
     "sha256": "2b7ba34e350a043c0b1190aa7a10e4c9ccc9d59bdc70a8557087fa86129f17ad",
@@ -3379,9 +3391,9 @@
   },
   "5bb4a95d-5a08-48eb-80db-4c3a63ec78a8": {
     "rule_name": "Suspicious PrintSpooler Service Executable File Creation",
-    "sha256": "84ef186fe1e107f4233f5b31bb8dbb4cc3d9164eda08868b2dcb9c41450e2ac7",
+    "sha256": "70177fc265fa2f24acad68cd0ef289816432b3766a1b8a43e6e4742eeb754522",
     "type": "new_terms",
-    "version": 317
+    "version": 318
   },
   "5bda8597-69a6-4b9e-87a2-69a7c963ea83": {
     "rule_name": "Boot File Copy",
@@ -3709,9 +3721,9 @@
   },
   "64f17c52-6c6e-479e-ba72-236f3df18f3d": {
     "rule_name": "Potential PowerShell Obfuscation via Invalid Escape Sequences",
-    "sha256": "2deaae9f306ec436dbcaa80ca7c8eedc5a563285015398e4017c49fdeabfa756",
+    "sha256": "fda6cdc3f42b88f38449c8dc374c2474384889313433b94cfc507f47fcf813c9",
     "type": "esql",
-    "version": 4
+    "version": 5
   },
   "6505e02e-28dd-41cd-b18f-64e649caa4e2": {
     "rule_name": "Manual Memory Dumping via Proc Filesystem",
@@ -3781,9 +3793,9 @@
   },
   "66883649-f908-4a5b-a1e0-54090a1d3a32": {
     "rule_name": "Connection to Commonly Abused Web Services",
-    "sha256": "6a43a05f6e5d1f479ce30211a8231a9e75a714f6cbcc39539e36e4ea0d69677b",
+    "sha256": "e0bcdab50088ca7a1827ec90afe4ec21cf937ffaf9b9069142b1709b1dae722d",
     "type": "eql",
-    "version": 120
+    "version": 121
   },
   "66c058f3-99f4-4d18-952b-43348f2577a0": {
     "rule_name": "Linux Process Hooking via GDB",
@@ -4075,9 +4087,9 @@
   },
   "6ea41894-66c3-4df7-ad6b-2c5074eb3df8": {
     "rule_name": "Potential Windows Error Manager Masquerading",
-    "sha256": "eec393cdeeee96acead27b0a15500be1195c020ebfdcc3d880d99c8583ce3e8b",
+    "sha256": "5c64c10228a0a54dc71ec736d0ceedf77938cee9b5bc4431aaa0997896c72131",
     "type": "eql",
-    "version": 213
+    "version": 214
   },
   "6ea55c81-e2ba-42f2-a134-bccf857ba922": {
     "rule_name": "Security Software Discovery using WMIC",
@@ -4766,9 +4778,9 @@
   },
   "818e23e6-2094-4f0e-8c01-22d30f3506c6": {
     "rule_name": "PowerShell Script Block Logging Disabled",
-    "sha256": "a74e2f1d576685aa6609e515d8f65b5beafaa71340e79e88d1d6c46e50c4ae67",
+    "sha256": "c21246a4390e985fe639c73d06b845ffd8a86744834565cfb9a614a61ebc0a22",
     "type": "eql",
-    "version": 312
+    "version": 313
   },
   "81cc58f5-8062-49a2-ba84-5cc4b4d31c40": {
     "rule_name": "Persistence via Kernel Module Modification",
@@ -4826,9 +4838,9 @@
   },
   "83bf249e-4348-47ba-9741-1202a09556ad": {
     "rule_name": "Suspicious Windows Powershell Arguments",
-    "sha256": "6a54429f392cbcfeb523e95780d8d88fba8ee94dec8f94a146586faccec92ba4",
+    "sha256": "d735d2babf46df807a11f9b74d63af45871886e7e814b0ebdcc72455f852dd6d",
     "type": "eql",
-    "version": 206
+    "version": 207
   },
   "83e9c2b3-24ef-4c1d-a8cd-5ebafb5dfa2f": {
     "rule_name": "Attempt to Disable IPTables or Firewall",
@@ -5865,9 +5877,9 @@
   },
   "9f432a8b-9588-4550-838e-1f77285580d3": {
     "rule_name": "Dynamic IEX Reconstruction via Method String Access",
-    "sha256": "23f848bcf8ab02b3323f34b311b522159a77a6bf97dcc3d8089023e82dd9f9d1",
+    "sha256": "d780db42a9137fadf25fea4f63c471704e7c6f0b488e4dbb61ceb66ce75e0efc",
     "type": "esql",
-    "version": 4
+    "version": 5
   },
   "9f962927-1a4f-45f3-a57b-287f2c7029c1": {
     "rule_name": "Potential Credential Access via DCSync",
@@ -6455,9 +6467,9 @@
   },
   "b0c98cfb-0745-4513-b6f9-08dddb033490": {
     "rule_name": "Potential Dynamic IEX Reconstruction via Environment Variables",
-    "sha256": "29e5db5ddaca083a914bfd531f068d353526cd492987ef80ced248ca1a8a5f29",
+    "sha256": "9107236bf5385a208a94f3b3a6934b5e38c8a96c3e94b398a2ca18dfc47a82c6",
     "type": "esql",
-    "version": 3
+    "version": 4
   },
   "b11116fd-023c-4718-aeb8-fa9d283fc53b": {
     "rule_name": "Kubeconfig File Creation or Modification",
@@ -6989,9 +7001,9 @@
   },
   "c18975f5-676c-4091-b626-81e8938aa2ee": {
     "rule_name": "Potential RemoteMonologue Attack",
-    "sha256": "5bfa9994c043217b1bfb42b4f0028e2871267f04b10dc7ba6898bc97a5f6551c",
+    "sha256": "f6b213b207b6c6bec26cd71b03f0737f031091f4392cb2de1ada95d48a1ed594",
     "type": "eql",
-    "version": 2
+    "version": 3
   },
   "c1a9ed70-d349-11ef-841c-f661ea17fbcd": {
     "rule_name": "Unusual AWS S3 Object Encryption with SSE-C",
@@ -7440,9 +7452,9 @@
   },
   "cd89602e-9db0-48e3-9391-ae3bf241acd8": {
     "rule_name": "MFA Deactivation with no Re-Activation for Okta User Account",
-    "sha256": "04ca550d18255b6f9e3437537b63cbdeedfe26f51c89cd8415e639ca6e57b68b",
+    "sha256": "ea5c43802417daa4603e8ddd5c129a8c63d3a5fc0bdf6ac8a481e2499dba26db",
     "type": "eql",
-    "version": 415
+    "version": 416
   },
   "cdbebdc1-dc97-43c6-a538-f26a20c0a911": {
     "rule_name": "Okta User Session Impersonation",
@@ -8420,9 +8432,9 @@
   },
   "e903ce9a-5ce6-4246-bb14-75ed3ec2edf5": {
     "rule_name": "Potential PowerShell Obfuscation via String Reordering",
-    "sha256": "61334267fab7a40c13164b761aa5542572e84f08266faa14e6282c22353baedb",
+    "sha256": "40bf0892c2068fff5e2b61f79cb7b0eedd5aaaa6193bd39a6eb188ef6184aac3",
     "type": "esql",
-    "version": 5
+    "version": 6
   },
   "e90ee3af-45fc-432e-a850-4a58cf14a457": {
     "rule_name": "High Number of Okta User Password Reset or Unlock Attempts",
@@ -8702,9 +8714,9 @@
   },
   "f036953a-4615-4707-a1ca-dc53bf69dcd5": {
     "rule_name": "Unusual Child Processes of RunDLL32",
-    "sha256": "5c086b3ea051770a44d257ef1b96a70801abf1965e2b5b1d1d4e54aaf3e033db",
+    "sha256": "b38b45cb340ce26c11c6845525f90bf3f24d61b736af9798d56249d3ab3547bd",
     "type": "eql",
-    "version": 211
+    "version": 212
   },
   "f0493cb4-9b15-43a9-9359-68c23a7f2cf3": {
     "rule_name": "Suspicious HTML File Creation",
@@ -8852,9 +8864,9 @@
   },
   "f38633f4-3b31-4c80-b13d-e77c70ce8254": {
     "rule_name": "Potential PowerShell Obfuscation via Reverse Keywords",
-    "sha256": "1a7bb59668aeb61b005ad82af62c813287c631d756892a3770a2eac56ca9102c",
+    "sha256": "4935469fc2fc470b586e4d5f9667f0e749fdc27c59dd87f33de369314ff2c9c4",
     "type": "esql",
-    "version": 3
+    "version": 4
   },
   "f391d3fd-219b-42a3-9ba9-2f66eb0155aa": {
     "rule_name": "Kill Command Execution",
@@ -9153,9 +9165,9 @@
   },
   "f9753455-8d55-4ad8-b70a-e07b6f18deea": {
     "rule_name": "Potential PowerShell Obfuscation via High Special Character Proportion",
-    "sha256": "54c9ab288e075807483eab23fbbea59aba7d8f760406d32755b0f297bbfe0810",
+    "sha256": "26098d2afb164e6f05a99cf24bd627301f808c5c1240693437cb14925bfab1c0",
     "type": "esql",
-    "version": 2
+    "version": 3
   },
   "f9790abf-bd0c-45f9-8b5f-d0b74015e029": {
     "rule_name": "Privileged Account Brute Force",
@@ -9171,9 +9183,9 @@
   },
   "f9abcddc-a05d-4345-a81d-000b79aa5525": {
     "rule_name": "Potential PowerShell Obfuscation via High Numeric Character Proportion",
-    "sha256": "014464fccb4a724e2e3fe5fcc79cc09c6d0fa696ee1d2d18d1a4ebe8c97ac533",
+    "sha256": "fa648e659bffe932aa1fffefe9c560668d631de9217505b3e3a7df813857b011",
     "type": "esql",
-    "version": 4
+    "version": 5
   },
   "fa01341d-6662-426b-9d0c-6d81e33c8a9d": {
     "rule_name": "Remote File Copy to a Hidden Share",

detection_rules/etc/version.lock.json

@@ -123,7 +123,6 @@ class RuleTomlEncoder(toml.TomlEncoder):  # type: ignore[reportMissingTypeArgume
     def __init__(self, *args: Any, **kwargs: Any) -> None:
         """Create the encoder but override some default functions."""
         super().__init__(*args, **kwargs)  # type: ignore[reportUnknownMemberType]
-        self._old_dump_str = toml.TomlEncoder().dump_funcs[str]
         self._old_dump_list = toml.TomlEncoder().dump_funcs[list]
         self.dump_funcs[str] = self.dump_str
         self.dump_funcs[str] = self.dump_str
@@ -148,10 +147,12 @@ def dump_str(self, v: str | NonformattedField) -> str:
         if multiline:
             if raw:
                 return "".join([TRIPLE_DQ, *initial_newline, *lines, TRIPLE_DQ])
-            return "\n".join([TRIPLE_SQ] + [self._old_dump_str(line)[1:-1] for line in lines] + [TRIPLE_SQ])
+            return "\n".join([TRIPLE_SQ] + [json.dumps(line)[1:-1] for line in lines] + [TRIPLE_SQ])
         if raw:
             return f"'{lines[0]:s}'"
-        return self._old_dump_str(v)
+        # In the toml library there is a magic replace for \\\\x -> u00 that we wish to avoid until #4979 is resolved
+        # Also addresses an issue where backslashes in certain strings are not properly escaped in self._old_dump_str(v)
+        return json.dumps(v)
 
     def _dump_flat_list(self, v: Iterable[Any]) -> str:
         """A slightly tweaked version of original dump_list, removing trailing commas."""

detection_rules/rule_formatter.py

@@ -63,6 +63,7 @@ coverage from the state of rules in the `main` branch.
 |[Elastic-detection-rules-tags-amazon-route53](https://mitre-attack.github.io/attack-navigator/#layerURL=https%3A%2F%2Fgist.githubusercontent.com%2Ftradebot-elastic%2F0443cfb5016bed103f1940b2f336e45a%2Fraw%2FElastic-detection-rules-tags-amazon-route53.json&leave_site_dialog=false&tabs=false)|
 |[Elastic-detection-rules-tags-amazon-s3](https://mitre-attack.github.io/attack-navigator/#layerURL=https%3A%2F%2Fgist.githubusercontent.com%2Ftradebot-elastic%2F0443cfb5016bed103f1940b2f336e45a%2Fraw%2FElastic-detection-rules-tags-amazon-s3.json&leave_site_dialog=false&tabs=false)|
 |[Elastic-detection-rules-tags-amazon-web-services](https://mitre-attack.github.io/attack-navigator/#layerURL=https%3A%2F%2Fgist.githubusercontent.com%2Ftradebot-elastic%2F0443cfb5016bed103f1940b2f336e45a%2Fraw%2FElastic-detection-rules-tags-amazon-web-services.json&leave_site_dialog=false&tabs=false)|
+|[Elastic-detection-rules-tags-api](https://mitre-attack.github.io/attack-navigator/#layerURL=https%3A%2F%2Fgist.githubusercontent.com%2Ftradebot-elastic%2F0443cfb5016bed103f1940b2f336e45a%2Fraw%2FElastic-detection-rules-tags-api.json&leave_site_dialog=false&tabs=false)|
 |[Elastic-detection-rules-tags-asset-visibility](https://mitre-attack.github.io/attack-navigator/#layerURL=https%3A%2F%2Fgist.githubusercontent.com%2Ftradebot-elastic%2F0443cfb5016bed103f1940b2f336e45a%2Fraw%2FElastic-detection-rules-tags-asset-visibility.json&leave_site_dialog=false&tabs=false)|
 |[Elastic-detection-rules-tags-auditd-manager](https://mitre-attack.github.io/attack-navigator/#layerURL=https%3A%2F%2Fgist.githubusercontent.com%2Ftradebot-elastic%2F0443cfb5016bed103f1940b2f336e45a%2Fraw%2FElastic-detection-rules-tags-auditd-manager.json&leave_site_dialog=false&tabs=false)|
 |[Elastic-detection-rules-tags-aws-cloudtrail](https://mitre-attack.github.io/attack-navigator/#layerURL=https%3A%2F%2Fgist.githubusercontent.com%2Ftradebot-elastic%2F0443cfb5016bed103f1940b2f336e45a%2Fraw%2FElastic-detection-rules-tags-aws-cloudtrail.json&leave_site_dialog=false&tabs=false)|

docs-dev/ATT&CK-coverage.md

@@ -1,6 +1,6 @@
 [project]
 name = "detection_rules"
-version = "1.3.23"
+version = "1.3.26"
 description = "Detection Rules is the home for rules used by Elastic Security. This repository is used for the development, maintenance, testing, validation, and release of rules for Elastic Security’s Detection Engine."
 readme = "README.md"
 requires-python = ">=3.12"