Merge branch 'main' into rt_0

Author: w0rk3r

CLI.md

@@ -87,7 +87,7 @@ and will accept any valid rule in the following formats:
 ```console
 Usage: detection_rules import-rules-to-repo [OPTIONS] [INPUT_FILE]...
 
-  Import rules from json, toml, yaml, or Kibana exported rule file(s).
+  Import rules from json, toml, or yaml files containing Kibana exported rule(s).
 
 Options:
   -ac, --action-connector-import  Include action connectors in export
@@ -102,6 +102,8 @@ Options:
   -ske, --skip-errors             Skip rule import errors
   -da, --default-author TEXT      Default author for rules missing one
   -snv, --strip-none-values       Strip None values from the rule
+  -lc, --local-creation-date      Preserve the local creation date of the rule
+  -lu, --local-updated-date       Preserve the local updated date of the rule
   -h, --help                      Show this message and exit.
 ```
 
@@ -145,16 +147,11 @@ Usage: detection_rules kibana [OPTIONS] COMMAND [ARGS]...
 
 Options:
   --ignore-ssl-errors TEXT
-  --space TEXT                 Kibana space
-  --provider-name TEXT         Elastic Cloud providers: cloud-basic and cloud-saml (for SSO)
-  --provider-type TEXT         Elastic Cloud providers: basic and saml (for SSO)
-  -ku, --kibana-user TEXT
-  --kibana-url TEXT
-  -kp, --kibana-password TEXT
-  -kc, --kibana-cookie TEXT    Cookie from an authed session
+  --space TEXT              Kibana space
   --api-key TEXT
-  --cloud-id TEXT              ID of the cloud instance.
-  -h, --help                   Show this message and exit.
+  --cloud-id TEXT           ID of the cloud instance.
+  --kibana-url TEXT
+  -h, --help                Show this message and exit.
 
 Commands:
   export-rules   Export custom rules from Kibana.
@@ -178,15 +175,10 @@ python -m detection_rules kibana search-alerts -h
 Kibana client:
 Options:
   --ignore-ssl-errors TEXT
-  --space TEXT                 Kibana space
-  --provider-name TEXT         Elastic Cloud providers: cloud-basic and cloud-saml (for SSO)
-  --provider-type TEXT         Elastic Cloud providers: basic and saml (for SSO)
-  -ku, --kibana-user TEXT
-  --kibana-url TEXT
-  -kp, --kibana-password TEXT
-  -kc, --kibana-cookie TEXT    Cookie from an authed session
+  --space TEXT              Kibana space
   --api-key TEXT
-  --cloud-id TEXT              ID of the cloud instance.
+  --cloud-id TEXT           ID of the cloud instance.
+  --kibana-url TEXT
 
 Usage: detection_rules kibana search-alerts [OPTIONS] [QUERY]
 
@@ -202,7 +194,7 @@ Options:
 ```
 
 Running the following command will print out a table showing any alerts that have been generated recently.
-`python3 -m detection_rules kibana --provider-name cloud-basic --kibana-url <url> --kibana-user <username> --kibana-password <password> search-alerts`
+`python3 -m detection_rules kibana --provider-name cloud-basic --kibana-url <url> --api-key <api-key> search-alerts`
 
 ```console
 
@@ -243,15 +235,10 @@ python -m detection_rules kibana import-rules -h
 Kibana client:
 Options:
   --ignore-ssl-errors TEXT
-  --space TEXT                 Kibana space
-  --provider-name TEXT         Elastic Cloud providers: cloud-basic and cloud-saml (for SSO)
-  --provider-type TEXT         Elastic Cloud providers: basic and saml (for SSO)
-  -ku, --kibana-user TEXT
-  --kibana-url TEXT
-  -kp, --kibana-password TEXT
-  -kc, --kibana-cookie TEXT    Cookie from an authed session
+  --space TEXT              Kibana space
   --api-key TEXT
-  --cloud-id TEXT              ID of the cloud instance.
+  --cloud-id TEXT           ID of the cloud instance.
+  --kibana-url TEXT
 
 Usage: detection_rules kibana import-rules [OPTIONS]
 
@@ -261,11 +248,11 @@ Options:
   -f, --rule-file FILE
   -d, --directory DIRECTORY       Recursively load rules from a directory
   -id, --rule-id TEXT
+  -nt, --no-tactic-filename       Allow rule filenames without tactic prefix. Use this if rules have been exported with this flag.
   -o, --overwrite                 Overwrite existing rules
   -e, --overwrite-exceptions      Overwrite exceptions in existing rules
   -ac, --overwrite-action-connectors
                                   Overwrite action connectors in existing rules
-  -nt, --no-tactic-filename       Allow rule filenames without tactic prefix. Use this if rules have been exported with this flag.
   -h, --help                      Show this message and exit.
 ```
 
@@ -422,14 +409,12 @@ Options:
   -f, --rule-file FILE
   -d, --directory DIRECTORY       Recursively load rules from a directory
   -id, --rule-id TEXT
+  -nt, --no-tactic-filename       Allow rule filenames without tactic prefix. Use this if rules have been exported with this flag.
   -o, --outfile PATH              Name of file for exported rules
   -r, --replace-id                Replace rule IDs with new IDs before export
-  --stack-version [7.8|7.9|7.10|7.11|7.12|7.13|7.14|7.15|7.16|8.0|8.1|8.2|8.3|8.4|8.5|8.6|8.7|8.8|8.9|8.10|8.11|8.12|8.13|8.14]
-                                  Downgrade a rule version to be compatible
-                                  with older instances of Kibana
-  -s, --skip-unsupported          If `--stack-version` is passed, skip rule
-                                  types which are unsupported (an error will
-                                  be raised otherwise)
+  --stack-version [7.8|7.9|7.10|7.11|7.12|7.13|7.14|7.15|7.16|8.0|8.1|8.2|8.3|8.4|8.5|8.6|8.7|8.8|8.9|8.10|8.11|8.12|8.13|8.14|8.15|8.16|8.17|8.18|9.0]
+                                  Downgrade a rule version to be compatible with older instances of Kibana
+  -s, --skip-unsupported          If `--stack-version` is passed, skip rule types which are unsupported (an error will be raised otherwise)
   --include-metadata              Add metadata to the exported rules
   -ac, --include-action-connectors
                                   Include Action Connectors in export
@@ -458,15 +443,10 @@ python -m detection_rules kibana upload-rule -h
 Kibana client:
 Options:
   --ignore-ssl-errors TEXT
-  --space TEXT                 Kibana space
-  --provider-name TEXT         Elastic Cloud providers: cloud-basic and cloud-saml (for SSO)
-  --provider-type TEXT         Elastic Cloud providers: basic and saml (for SSO)
-  -ku, --kibana-user TEXT
-  --kibana-url TEXT
-  -kp, --kibana-password TEXT
-  -kc, --kibana-cookie TEXT    Cookie from an authed session
+  --space TEXT              Kibana space
   --api-key TEXT
-  --cloud-id TEXT              ID of the cloud instance.
+  --cloud-id TEXT           ID of the cloud instance.
+  --kibana-url TEXT
 
 Usage: detection_rules kibana upload-rule [OPTIONS]
 
@@ -476,6 +456,7 @@ Options:
   -f, --rule-file FILE
   -d, --directory DIRECTORY  Recursively load rules from a directory
   -id, --rule-id TEXT
+  -nt, --no-tactic-filename  Allow rule filenames without tactic prefix. Use this if rules have been exported with this flag.
   -r, --replace-id           Replace rule IDs with new IDs before export
   -h, --help                 Show this message and exit.
 ```
@@ -484,6 +465,8 @@ Options:
 
 This command should be run with the `CUSTOM_RULES_DIR` envvar set, that way proper validation is applied to versioning when the rules are downloaded. See the [custom rules docs](docs-dev/custom-rules-management.md) for more information.
 
+Note: This command can be used for exporting pre-built, customized pre-built, and custom rules. By default, all rules will be exported. Use the `-cro` flag to only export custom rules, or the `-eq` flag to filter by query.
+
 ```
 python -m detection_rules kibana export-rules -h
 
@@ -494,15 +477,10 @@ python -m detection_rules kibana export-rules -h
 Kibana client:
 Options:
   --ignore-ssl-errors TEXT
-  --space TEXT                 Kibana space
-  --provider-name TEXT         Elastic Cloud providers: cloud-basic and cloud-saml (for SSO)
-  --provider-type TEXT         Elastic Cloud providers: basic and saml (for SSO)
-  -ku, --kibana-user TEXT
-  --kibana-url TEXT
-  -kp, --kibana-password TEXT
-  -kc, --kibana-cookie TEXT    Cookie from an authed session
+  --space TEXT              Kibana space
   --api-key TEXT
-  --cloud-id TEXT              ID of the cloud instance.
+  --cloud-id TEXT           ID of the cloud instance.
+  --kibana-url TEXT
 
 Usage: detection_rules kibana export-rules [OPTIONS]
 
@@ -523,6 +501,10 @@ Options:
   -s, --skip-errors               Skip errors when exporting rules
   -sv, --strip-version            Strip the version fields from all rules
   -nt, --no-tactic-filename       Exclude tactic prefix in exported filenames for rules. Use same flag for import-rules to prevent warnings and disable its unit test.
+  -lc, --local-creation-date      Preserve the local creation date of the rule
+  -lu, --local-updated-date       Preserve the local updated date of the rule
+  -cro, --custom-rules-only       Only export custom rules
+  -eq, --export-query TEXT        Apply a query filter to exporting rules e.g. "alert.attributes.tags: \"test\"" to filter for rules that have the tag "test"
   -h, --help                      Show this message and exit.
 
 ```

Makefile

@@ -2,6 +2,7 @@
 ### detection-rules
 #################
 
+APP_NAME := detection-rules
 VENV := ./env/detection-rules-build
 VENV_BIN := $(VENV)/bin
 PYTHON := $(VENV_BIN)/python
@@ -26,7 +27,7 @@ deps: $(VENV)
 	$(PIP) install lib/kql
 
 .PHONY: hunting-deps
-deps: $(VENV)
+hunting-deps: $(VENV)
 	@echo "Installing all dependencies..."
 	$(PIP) install .[hunting]
 
@@ -58,13 +59,13 @@ test-remote-cli: $(VENV) deps
 	@./detection_rules/etc/test_remote_cli.bash
 
 .PHONY: test-hunting-cli
-test-remote-cli: $(VENV) hunting-deps
+test-hunting-cli: $(VENV) hunting-deps
 	@echo "Executing test_hunting_cli script..."
 	@./detection_rules/etc/test_hunting_cli.bash
 
 .PHONY: release
 release: deps
-	@echo "RELEASE: $(app_name)"
+	@echo "RELEASE: $(APP_NAME)"
 	$(PYTHON) -m detection_rules dev build-release --generate-navigator
 	rm -rf dist
 	mkdir dist

README.md

@@ -103,23 +103,29 @@ Usage: detection_rules [OPTIONS] COMMAND [ARGS]...
   Commands for detection-rules repository.
 
 Options:
-  -d, --debug / -n, --no-debug  Print full exception stacktrace on errors
+  -D, --debug / -N, --no-debug  Print full exception stacktrace on errors
   -h, --help                    Show this message and exit.
 
 Commands:
-  create-rule     Create a detection rule.
-  dev             Commands for development and management by internal...
-  es              Commands for integrating with Elasticsearch.
-  import-rules    Import rules from json, toml, or Kibana exported rule...
-  kibana          Commands for integrating with Kibana.
-  mass-update     Update multiple rules based on eql results.
-  normalize-data  Normalize Elasticsearch data timestamps and sort.
-  rule-search     Use KQL or EQL to find matching rules.
-  test            Run unit tests over all of the rules.
-  toml-lint       Cleanup files with some simple toml formatting.
-  validate-all    Check if all rules validates against a schema.
-  validate-rule   Check if a rule staged in rules dir validates against a...
-  view-rule       View an internal rule or specified rule file.
+  build-limited-rules     Import rules from json, toml, or Kibana exported rule file(s), filter out unsupported ones, and write to output NDJSON file.
+  build-threat-map-entry  Build a threat map entry.
+  create-rule             Create a detection rule.
+  custom-rules            Commands for supporting custom rules.
+  dev                     Commands related to the Elastic Stack rules release lifecycle.
+  es                      Commands for integrating with Elasticsearch.
+  export-rules-from-repo  Export rule(s) and exception(s) into an importable ndjson file.
+  generate-rules-index    Generate enriched indexes of rules, based on a KQL search, for indexing/importing into elasticsearch/kibana.
+  import-rules-to-repo    Import rules from json, toml, or yaml files containing Kibana exported rule(s).
+  kibana                  Commands for integrating with Kibana.
+  mass-update             Update multiple rules based on eql results.
+  normalize-data          Normalize Elasticsearch data timestamps and sort.
+  rule-search             Use KQL or EQL to find matching rules.
+  test                    Run unit tests over all of the rules.
+  toml-lint               Cleanup files with some simple toml formatting.
+  typosquat               Commands for generating typosquat detections.
+  validate-all            Check if all rules validates against a schema.
+  validate-rule           Check if a rule staged in rules dir validates against a schema.
+  view-rule               View an internal rule or specified rule file.
 ```
 
 Note:

detection_rules/etc/test_hunting_cli.bash

@@ -15,11 +15,11 @@ echo "Refreshing index"
 python -m hunting refresh-index
 
 echo "Generating Markdown: initial_access_higher_than_average_failed_authentication.toml"
-python -m hunting generate-markdown /Users/tdejesus/code/src/detection-rules/hunting/okta/queries/initial_access_higher_than_average_failed_authentication.toml
+python -m hunting generate-markdown hunting/okta/queries/initial_access_higher_than_average_failed_authentication.toml
 
 echo "Running Query: low_volume_external_network_connections_from_process.toml"
 echo "Requires .detection-rules-cfg.json credentials file set."
-python -m hunting run-query --file-path /Users/tdejesus/code/src/detection-rules/hunting/linux/queries/low_volume_external_network_connections_from_process.toml --all
+python -m hunting run-query --file-path hunting/linux/queries/low_volume_external_network_connections_from_process.toml --all
 
 echo "Viewing Hunt: 12526f14-5e35-4f5f-884c-96c6a353a544"
 python -m hunting view-hunt --uuid 12526f14-5e35-4f5f-884c-96c6a353a544 --format json

hunting/markdown.py

@@ -93,7 +93,7 @@ def update_or_add_entry(self, hunt_config: Hunt, toml_path: Path) -> None:
 
         entry = {
             'name': hunt_config.name,
-            'path': f"./{toml_path.relative_to(self.base_path).as_posix()}",
+            'path': f"./{toml_path.resolve().relative_to(self.base_path).as_posix()}",
             'mitre': hunt_config.mitre
         }
 

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "detection_rules"
-version = "1.2.13"
+version = "1.2.15"
 description = "Detection Rules is the home for rules used by Elastic Security. This repository is used for the development, maintenance, testing, validation, and release of rules for Elastic Security’s Detection Engine."
 readme = "README.md"
 requires-python = ">=3.12"

rules/windows/execution_downloaded_url_file.toml

@@ -2,7 +2,7 @@
 creation_date = "2020/09/02"
 integration = ["endpoint"]
 maturity = "production"
-updated_date = "2025/01/15"
+updated_date = "2025/06/11"
 
 [rule]
 author = ["Elastic"]
@@ -31,7 +31,7 @@ type = "eql"
 
 query = '''
 file where host.os.type == "windows" and event.type == "creation" and file.extension == "url"
-   and file.Ext.windows.zone_identifier > 1 and not process.name : "explorer.exe"
+   and file.Ext.windows.zone_identifier == 3 
 '''
 note = """## Triage and analysis