Skip to content

Commit a80242c

Browse files
terrancedejesusterrancedejesus
authored
Merge branch 'main' into terrancedejesus/issue5216
2 parents 2a7c516 + c724631 commit a80242c

29 files changed

+1972
-407
lines changed

‎.github/workflows/esql-validation.yml‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
name: ES|QL Validation
2+
on:
3+
pull_request:
4+
branches: [ "*" ]
5+
paths:
6+
- 'rules/**/*.toml'
7+
jobs:
8+
build-and-validate:
9+
runs-on: ubuntu-latest
10+
11+
steps:
12+
- name: Setup Detection Rules
13+
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5
14+
with:
15+
fetch-depth: 0
16+
path: detection-rules
17+
18+
- name: Check if new or modified rule files are ESQL rules
19+
id: check-esql
20+
run: |
21+
cd detection-rules
22+
23+
# Check if the event is a push
24+
if [ "${{ github.event_name }}" = "push" ]; then
25+
echo "Triggered by a push event. Setting run_esql=true."
26+
echo "run_esql=true" >> $GITHUB_ENV
27+
exit 0
28+
fi
29+
30+
MODIFIED_FILES=$(git diff --name-only --diff-filter=AM HEAD~1 | grep '^rules/.*\.toml$' || true)
31+
if [ -z "$MODIFIED_FILES" ]; then
32+
echo "No modified or new .toml files found. Skipping workflow."
33+
echo "run_esql=false" >> $GITHUB_ENV
34+
exit 0
35+
fi
36+
37+
if ! grep -q 'type = "esql"' $MODIFIED_FILES; then
38+
echo "No 'type = \"esql\"' found in the modified .toml files. Skipping workflow."
39+
echo "run_esql=false" >> $GITHUB_ENV
40+
exit 0
41+
fi
42+
43+
echo "run_esql=true" >> $GITHUB_ENV
44+
45+
- name: Check out repository
46+
env:
47+
DR_CLOUD_ID: ${{ secrets.dr_cloud_id }}
48+
DR_API_KEY: ${{ secrets.dr_api_key }}
49+
if: ${{ !env.DR_CLOUD_ID && !env.DR_API_KEY && env.run_esql == 'true' }}
50+
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5
51+
with:
52+
path: elastic-container
53+
repository: peasead/elastic-container
54+
55+
- name: Build and run containers
56+
env:
57+
DR_CLOUD_ID: ${{ secrets.dr_cloud_id }}
58+
DR_API_KEY: ${{ secrets.dr_api_key }}
59+
if: ${{ !env.DR_CLOUD_ID && !env.DR_API_KEY && env.run_esql == 'true' }}
60+
run: |
61+
cd elastic-container
62+
GENERATED_PASSWORD=$(openssl rand -base64 16)
63+
sed -i "s|changeme|$GENERATED_PASSWORD|" .env
64+
echo "::add-mask::$GENERATED_PASSWORD"
65+
echo "GENERATED_PASSWORD=$GENERATED_PASSWORD" >> $GITHUB_ENV
66+
set -x
67+
bash elastic-container.sh start
68+
69+
- name: Get API Key and setup auth
70+
env:
71+
DR_CLOUD_ID: ${{ secrets.dr_cloud_id }}
72+
DR_API_KEY: ${{ secrets.dr_api_key }}
73+
DR_ELASTICSEARCH_URL: "https://localhost:9200"
74+
ES_USER: "elastic"
75+
ES_PASSWORD: ${{ env.GENERATED_PASSWORD }}
76+
if: ${{ !env.DR_CLOUD_ID && !env.DR_API_KEY && env.run_esql == 'true' }}
77+
run: |
78+
cd detection-rules
79+
response=$(curl -k -X POST -u "$ES_USER:$ES_PASSWORD" -H "Content-Type: application/json" -d '{
80+
"name": "tmp-api-key",
81+
"expiration": "1d"
82+
}' "$DR_ELASTICSEARCH_URL/_security/api_key")
83+
84+
DR_API_KEY=$(echo "$response" | jq -r '.encoded')
85+
echo "::add-mask::$DR_API_KEY"
86+
echo "DR_API_KEY=$DR_API_KEY" >> $GITHUB_ENV
87+
88+
- name: Set up Python 3.13
89+
if: ${{ env.run_esql == 'true' }}
90+
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6
91+
with:
92+
python-version: '3.13'
93+
94+
- name: Install dependencies
95+
if: ${{ env.run_esql == 'true' }}
96+
run: |
97+
cd detection-rules
98+
python -m pip install --upgrade pip
99+
pip cache purge
100+
pip install .[dev]
101+
102+
- name: Remote Test ESQL Rules
103+
if: ${{ env.run_esql == 'true' }}
104+
env:
105+
DR_CLOUD_ID: ${{ secrets.dr_cloud_id || '' }}
106+
DR_KIBANA_URL: ${{ secrets.dr_cloud_id == '' && 'https://localhost:5601' || '' }}
107+
DR_ELASTICSEARCH_URL: ${{ secrets.dr_cloud_id == '' && 'https://localhost:9200' || '' }}
108+
DR_API_KEY: ${{ secrets.dr_api_key || env.DR_API_KEY }}
109+
DR_IGNORE_SSL_ERRORS: ${{ secrets.dr_cloud_id == '' && 'true' || '' }}
110+
run: |
111+
cd detection-rules
112+
python -m detection_rules dev test esql-remote-validation

‎.github/workflows/pythonpackage.yml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,13 @@ jobs:
3737
env:
3838
# only run the test test_rule_change_has_updated_date on pull request events to main
3939
GITHUB_EVENT_NAME: "${{ github.event_name}}"
40+
# only run remote validation if repo is set to do so otherwise defer to .github/workflows/esql-validation.yml
41+
DR_REMOTE_ESQL_VALIDATION: "${{ vars.remote_esql_validation }}"
42+
DR_CLOUD_ID: ${{ secrets.dr_cloud_id }}
43+
DR_KIBANA_URL: ${{ secrets.dr_cloud_id }}
44+
DR_ELASTICSEARCH_URL: ${{ secrets.dr_cloud_id }}
45+
DR_API_KEY: ${{ secrets.dr_api_key }}
46+
DR_IGNORE_SSL_ERRORS: ${{ secrets.dr_cloud_id }}
4047
run: |
4148
python -m detection_rules test
4249

‎CLI.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,10 @@ Using the environment variable `DR_BYPASS_TIMELINE_TEMPLATE_VALIDATION` will byp
4949
Using the environment variable `DR_CLI_MAX_WIDTH` will set a custom max width for the click CLI.
5050
For instance, some users may want to increase the default value in cases where help messages are cut off.
5151

52+
Using the environment variable `DR_REMOTE_ESQL_VALIDATION` will enable remote ESQL validation for rules that use ESQL queries. This validation will be performed whenever the rule is loaded including for example the view-rule command. This requires the appropriate kibana_url or cloud_id, api_key, and es_url to be set in the config file or as environment variables.
53+
54+
Using the environment variable `DR_SKIP_EMPTY_INDEX_CLEANUP` will disable the cleanup of remote testing indexes that are created as part of the remote ESQL validation. By default, these indexes are deleted after the validation is complete, or upon validation error.
55+
5256
## Importing rules into the repo
5357

5458
You can import rules into the repo using the `create-rule` or `import-rules-to-repo` commands. Both of these commands will

‎detection_rules/cli_utils.py‎

Lines changed: 101 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,10 @@
77
import datetime
88
import functools
99
import os
10+
import re
11+
import time
1012
import typing
13+
import uuid
1114
from collections.abc import Callable
1215
from pathlib import Path
1316
from typing import Any
@@ -27,6 +30,104 @@
2730
RULES_CONFIG = parse_rules_config()
2831

2932

33+
def schema_prompt(name: str, value: Any | None = None, is_required: bool = False, **options: Any) -> Any: # noqa: PLR0911, PLR0912, PLR0915
34+
"""Interactively prompt based on schema requirements."""
35+
field_type = options.get("type")
36+
pattern: str | None = options.get("pattern")
37+
enum = options.get("enum", [])
38+
minimum = int(options["minimum"]) if "minimum" in options else None
39+
maximum = int(options["maximum"]) if "maximum" in options else None
40+
min_item = int(options.get("min_items", 0))
41+
max_items = int(options.get("max_items", 9999))
42+
43+
default = options.get("default")
44+
if default is not None and str(default).lower() in ("true", "false"):
45+
default = str(default).lower()
46+
47+
if "date" in name:
48+
default = time.strftime("%Y/%m/%d")
49+
50+
if name == "rule_id":
51+
default = str(uuid.uuid4())
52+
53+
if len(enum) == 1 and is_required and field_type not in ("array", ["array"]):
54+
return enum[0]
55+
56+
def _check_type(_val: Any) -> bool: # noqa: PLR0911
57+
if field_type in ("number", "integer") and not str(_val).isdigit():
58+
print(f"Number expected but got: {_val}")
59+
return False
60+
if pattern:
61+
match = re.match(pattern, _val)
62+
if not match or len(match.group(0)) != len(_val):
63+
print(f"{_val} did not match pattern: {pattern}!")
64+
return False
65+
if enum and _val not in enum:
66+
print("{} not in valid options: {}".format(_val, ", ".join(enum)))
67+
return False
68+
if minimum and (type(_val) is int and int(_val) < minimum):
69+
print(f"{_val!s} is less than the minimum: {minimum!s}")
70+
return False
71+
if maximum and (type(_val) is int and int(_val) > maximum):
72+
print(f"{_val!s} is greater than the maximum: {maximum!s}")
73+
return False
74+
if type(_val) is str and field_type == "boolean" and _val.lower() not in ("true", "false"):
75+
print(f"Boolean expected but got: {_val!s}")
76+
return False
77+
return True
78+
79+
def _convert_type(_val: Any) -> Any:
80+
if field_type == "boolean" and type(_val) is not bool:
81+
_val = _val.lower() == "true"
82+
return int(_val) if field_type in ("number", "integer") else _val
83+
84+
prompt = (
85+
"{name}{default}{required}{multi}".format(
86+
name=name,
87+
default=f' [{default}] ("n/a" to leave blank) ' if default else "",
88+
required=" (required) " if is_required else "",
89+
multi=(" (multi, comma separated) " if field_type in ("array", ["array"]) else ""),
90+
).strip()
91+
+ ": "
92+
)
93+
94+
while True:
95+
result = value or input(prompt) or default
96+
if result == "n/a":
97+
result = None
98+
99+
if not result:
100+
if is_required:
101+
value = None
102+
continue
103+
return None
104+
105+
if field_type in ("array", ["array"]):
106+
result_list = result.split(",")
107+
108+
if not (min_item < len(result_list) < max_items):
109+
if is_required:
110+
value = None
111+
break
112+
return []
113+
114+
for value in result_list:
115+
if not _check_type(value):
116+
if is_required:
117+
value = None # noqa: PLW2901
118+
break
119+
return []
120+
if is_required and value is None:
121+
continue
122+
return [_convert_type(r) for r in result_list]
123+
if _check_type(result):
124+
return _convert_type(result)
125+
if is_required:
126+
value = None
127+
continue
128+
return None
129+
130+
30131
def single_collection(f: Callable[..., Any]) -> Callable[..., Any]:
31132
"""Add arguments to get a RuleCollection by file, directory or a list of IDs"""
32133
from .misc import raise_client_error
@@ -145,7 +246,6 @@ def rule_prompt( # noqa: PLR0912, PLR0913, PLR0915
145246
**kwargs: Any,
146247
) -> TOMLRule | str:
147248
"""Prompt loop to build a rule."""
148-
from .misc import schema_prompt
149249

150250
additional_required = additional_required or []
151251
creation_date = datetime.date.today().strftime("%Y/%m/%d") # noqa: DTZ011

‎detection_rules/devtools.py‎

Lines changed: 79 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,8 @@
2525
import pytoml # type: ignore[reportMissingTypeStubs]
2626
import requests.exceptions
2727
import yaml
28-
from elasticsearch import Elasticsearch
28+
from elasticsearch import BadRequestError, Elasticsearch
29+
from elasticsearch import ConnectionError as ESConnectionError
2930
from eql.table import Table # type: ignore[reportMissingTypeStubs]
3031
from eql.utils import load_dump # type: ignore[reportMissingTypeStubs, reportUnknownVariableType]
3132
from kibana.connector import Kibana # type: ignore[reportMissingTypeStubs]
@@ -39,6 +40,9 @@
3940
from .docs import REPO_DOCS_DIR, IntegrationSecurityDocs, IntegrationSecurityDocsMDX
4041
from .ecs import download_endpoint_schemas, download_schemas
4142
from .endgame import EndgameSchemaManager
43+
from .esql_errors import (
44+
ESQL_EXCEPTION_TYPES,
45+
)
4246
from .eswrap import CollectEvents, add_range_to_dsl
4347
from .ghwrap import GithubClient, update_gist
4448
from .integrations import (
@@ -50,7 +54,13 @@
5054
load_integrations_manifests,
5155
)
5256
from .main import root
53-
from .misc import PYTHON_LICENSE, add_client, raise_client_error
57+
from .misc import (
58+
PYTHON_LICENSE,
59+
add_client,
60+
get_default_elasticsearch_client,
61+
get_default_kibana_client,
62+
raise_client_error,
63+
)
5464
from .packaging import CURRENT_RELEASE_PATH, PACKAGE_FILE, RELEASE_DIR, Package
5565
from .rule import (
5666
AnyRuleData,
@@ -63,6 +73,7 @@
6373
TOMLRuleContents,
6474
)
6575
from .rule_loader import RuleCollection, production_filter
76+
from .rule_validators import ESQLValidator
6677
from .schemas import definitions, get_stack_versions
6778
from .utils import check_version_lock_double_bumps, dict_hash, get_etc_path, get_path
6879
from .version_lock import VersionLockFile, loaded_version_lock
@@ -1403,6 +1414,72 @@ def rule_event_search( # noqa: PLR0913
14031414
raise_client_error("Rule is not a query rule!")
14041415

14051416

1417+
@test_group.command("esql-remote-validation")
1418+
@click.option(
1419+
"--verbosity",
1420+
type=click.IntRange(0, 1),
1421+
default=0,
1422+
help="Set verbosity level: 0 for minimal output, 1 for detailed output.",
1423+
)
1424+
def esql_remote_validation(
1425+
verbosity: int,
1426+
) -> None:
1427+
"""Search using a rule file against an Elasticsearch instance."""
1428+
1429+
rule_collection: RuleCollection = RuleCollection.default().filter(production_filter)
1430+
esql_rules = [r for r in rule_collection if r.contents.data.type == "esql"]
1431+
1432+
click.echo(f"ESQL rules loaded: {len(esql_rules)}")
1433+
1434+
if not esql_rules:
1435+
return
1436+
# TODO(eric-forte-elastic): @add_client https://github.com/elastic/detection-rules/issues/5156 # noqa: FIX002
1437+
with get_default_kibana_client() as kibana_client, get_default_elasticsearch_client() as elastic_client:
1438+
if not kibana_client or not elastic_client:
1439+
raise_client_error("Skipping remote validation due to missing client")
1440+
1441+
failed_count = 0
1442+
fail_list: list[str] = []
1443+
max_retries = 3
1444+
for r in esql_rules:
1445+
retry_count = 0
1446+
while retry_count < max_retries:
1447+
try:
1448+
validator = ESQLValidator(r.contents.data.query) # type: ignore[reportIncompatibleMethodOverride]
1449+
_ = validator.remote_validate_rule_contents(kibana_client, elastic_client, r.contents, verbosity)
1450+
break
1451+
except (ValueError, BadRequestError, *ESQL_EXCEPTION_TYPES) as e: # type: ignore[reportUnknownMemberType]
1452+
e_type = type(e) # type: ignore[reportUnknownMemberType]
1453+
if isinstance(e, ESQL_EXCEPTION_TYPES):
1454+
click.echo(click.style(f"{r.contents.data.rule_id} ", fg="red", bold=True), nl=False)
1455+
_ = e.show() # type: ignore[reportUnknownMemberType]
1456+
else:
1457+
click.echo(f"FAILURE: {e_type}: {e}") # type: ignore[reportUnknownMemberType]
1458+
fail_list.append(f"{r.contents.data.rule_id} FAILURE: {e_type}: {e}") # type: ignore[reportUnknownMemberType]
1459+
failed_count += 1
1460+
break
1461+
except ESConnectionError as e:
1462+
retry_count += 1
1463+
click.echo(f"Connection error: {e}. Retrying {retry_count}/{max_retries}...")
1464+
time.sleep(30)
1465+
if retry_count == max_retries:
1466+
click.echo(f"FAILURE: {e} after {max_retries} retries")
1467+
fail_list.append(f"FAILURE: {e} after {max_retries} retries")
1468+
failed_count += 1
1469+
1470+
click.echo(f"Total rules: {len(esql_rules)}")
1471+
click.echo(f"Failed rules: {failed_count}")
1472+
1473+
_ = Path("failed_rules.log").write_text("\n".join(fail_list), encoding="utf-8")
1474+
click.echo("Failed rules written to failed_rules.log")
1475+
if failed_count > 0:
1476+
click.echo("Failed rule IDs:")
1477+
uuids = {line.split()[0] for line in fail_list}
1478+
click.echo("\n".join(uuids))
1479+
ctx = click.get_current_context()
1480+
ctx.exit(1)
1481+
1482+
14061483
@test_group.command("rule-survey")
14071484
@click.argument("query", required=False)
14081485
@click.option("--date-range", "-d", type=(str, str), default=("now-7d", "now"), help="Date range to scope search")

0 commit comments

Comments
 (0)