Skip to content

Commit a1670a2

Browse files
w0rk3rw0rk3r
authored
Merge branch 'main' into docs_windows_audit
2 parents 7d59bcf + 62d7316 commit a1670a2

File tree

281 files changed

+12069
-2171
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

281 files changed

+12069
-2171
lines changed

‎.github/workflows/esql-validation.yml‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
name: ES|QL Validation
2+
on:
3+
pull_request:
4+
branches: [ "*" ]
5+
jobs:
6+
build-and-validate:
7+
runs-on: ubuntu-latest
8+
9+
steps:
10+
- name: Setup Detection Rules
11+
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5
12+
with:
13+
fetch-depth: 0
14+
path: detection-rules
15+
16+
- name: Check if new or modified rule files are ESQL rules
17+
id: check-esql
18+
run: |
19+
cd detection-rules
20+
21+
# Check if the event is a push
22+
if [ "${{ github.event_name }}" = "push" ]; then
23+
echo "Triggered by a push event. Setting run_esql=true."
24+
echo "run_esql=true" >> $GITHUB_ENV
25+
exit 0
26+
fi
27+
28+
MODIFIED_FILES=$(git diff --name-only --diff-filter=AM HEAD~1 | grep '^rules/.*\.toml$' || true)
29+
if [ -z "$MODIFIED_FILES" ]; then
30+
echo "No modified or new .toml files found. Skipping workflow."
31+
echo "run_esql=false" >> $GITHUB_ENV
32+
exit 0
33+
fi
34+
35+
if ! grep -q 'type = "esql"' $MODIFIED_FILES; then
36+
echo "No 'type = \"esql\"' found in the modified .toml files. Skipping workflow."
37+
echo "run_esql=false" >> $GITHUB_ENV
38+
exit 0
39+
fi
40+
41+
echo "run_esql=true" >> $GITHUB_ENV
42+
43+
- name: Check out repository
44+
env:
45+
DR_CLOUD_ID: ${{ secrets.dr_cloud_id }}
46+
DR_API_KEY: ${{ secrets.dr_api_key }}
47+
if: ${{ !env.DR_CLOUD_ID && !env.DR_API_KEY && env.run_esql == 'true' }}
48+
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5
49+
with:
50+
path: elastic-container
51+
repository: peasead/elastic-container
52+
53+
- name: Build and run containers
54+
env:
55+
DR_CLOUD_ID: ${{ secrets.dr_cloud_id }}
56+
DR_API_KEY: ${{ secrets.dr_api_key }}
57+
if: ${{ !env.DR_CLOUD_ID && !env.DR_API_KEY && env.run_esql == 'true' }}
58+
run: |
59+
cd elastic-container
60+
GENERATED_PASSWORD=$(openssl rand -base64 16)
61+
sed -i "s|changeme|$GENERATED_PASSWORD|" .env
62+
echo "::add-mask::$GENERATED_PASSWORD"
63+
echo "GENERATED_PASSWORD=$GENERATED_PASSWORD" >> $GITHUB_ENV
64+
set -x
65+
bash elastic-container.sh start
66+
67+
- name: Get API Key and setup auth
68+
env:
69+
DR_CLOUD_ID: ${{ secrets.dr_cloud_id }}
70+
DR_API_KEY: ${{ secrets.dr_api_key }}
71+
DR_ELASTICSEARCH_URL: "https://localhost:9200"
72+
ES_USER: "elastic"
73+
ES_PASSWORD: ${{ env.GENERATED_PASSWORD }}
74+
if: ${{ !env.DR_CLOUD_ID && !env.DR_API_KEY && env.run_esql == 'true' }}
75+
run: |
76+
cd detection-rules
77+
response=$(curl -k -X POST -u "$ES_USER:$ES_PASSWORD" -H "Content-Type: application/json" -d '{
78+
"name": "tmp-api-key",
79+
"expiration": "1d"
80+
}' "$DR_ELASTICSEARCH_URL/_security/api_key")
81+
82+
DR_API_KEY=$(echo "$response" | jq -r '.encoded')
83+
echo "::add-mask::$DR_API_KEY"
84+
echo "DR_API_KEY=$DR_API_KEY" >> $GITHUB_ENV
85+
86+
- name: Set up Python 3.13
87+
if: ${{ env.run_esql == 'true' }}
88+
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6
89+
with:
90+
python-version: '3.13'
91+
92+
- name: Install dependencies
93+
if: ${{ env.run_esql == 'true' }}
94+
run: |
95+
cd detection-rules
96+
python -m pip install --upgrade pip
97+
pip cache purge
98+
pip install .[dev]
99+
100+
- name: Remote Test ESQL Rules
101+
if: ${{ env.run_esql == 'true' }}
102+
env:
103+
DR_CLOUD_ID: ${{ secrets.dr_cloud_id || '' }}
104+
DR_KIBANA_URL: ${{ secrets.dr_cloud_id == '' && 'https://localhost:5601' || '' }}
105+
DR_ELASTICSEARCH_URL: ${{ secrets.dr_cloud_id == '' && 'https://localhost:9200' || '' }}
106+
DR_API_KEY: ${{ secrets.dr_api_key || env.DR_API_KEY }}
107+
DR_IGNORE_SSL_ERRORS: ${{ secrets.dr_cloud_id == '' && 'true' || '' }}
108+
run: |
109+
cd detection-rules
110+
python -m detection_rules dev test esql-remote-validation

‎.github/workflows/kibana-mitre-update.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ jobs:
1818

1919
- name: Get MITRE Attack changed files
2020
id: changed-attack-files
21-
uses: tj-actions/changed-files@2f7c5bfce28377bc069a65ba478de0a74aa0ca32 # v46.0.1
21+
uses: tj-actions/changed-files@ed68ef82c095e0d48ec87eccea555d944a631a4c # v46.0.5
2222
with:
2323
files: detection_rules/etc/attack-v*.json.gz
2424

‎.github/workflows/lock-versions.yml‎

Lines changed: 57 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ on:
66
description: 'List of branches to lock versions (ordered, comma separated)'
77
required: true
88
# 7.17 was intentionally skipped because it was added late and was bug fix only
9-
default: '8.18,8.19,9.0,9.1'
9+
default: '8.19,9.0,9.1,9.2'
1010

1111
jobs:
1212
pr:
@@ -37,7 +37,57 @@ jobs:
3737
pip cache purge
3838
pip install .[dev]
3939
40+
- name: Check out container repository
41+
env:
42+
DR_CLOUD_ID: ${{ secrets.dr_cloud_id }}
43+
DR_API_KEY: ${{ secrets.dr_api_key }}
44+
if: ${{ !env.DR_CLOUD_ID && !env.DR_API_KEY }}
45+
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5
46+
with:
47+
path: elastic-container
48+
repository: peasead/elastic-container
49+
50+
- name: Build and run containers
51+
env:
52+
DR_CLOUD_ID: ${{ secrets.dr_cloud_id }}
53+
DR_API_KEY: ${{ secrets.dr_api_key }}
54+
if: ${{ !env.DR_CLOUD_ID && !env.DR_API_KEY }}
55+
run: |
56+
cd elastic-container
57+
GENERATED_PASSWORD=$(openssl rand -base64 16)
58+
sed -i "s|changeme|$GENERATED_PASSWORD|" .env
59+
echo "::add-mask::$GENERATED_PASSWORD"
60+
echo "GENERATED_PASSWORD=$GENERATED_PASSWORD" >> $GITHUB_ENV
61+
set -x
62+
bash elastic-container.sh start
63+
64+
- name: Get API Key and setup auth
65+
env:
66+
DR_CLOUD_ID: ${{ secrets.dr_cloud_id }}
67+
DR_API_KEY: ${{ secrets.dr_api_key }}
68+
DR_ELASTICSEARCH_URL: "https://localhost:9200"
69+
ES_USER: "elastic"
70+
ES_PASSWORD: ${{ env.GENERATED_PASSWORD }}
71+
if: ${{ !env.DR_CLOUD_ID && !env.DR_API_KEY }}
72+
run: |
73+
cd detection-rules
74+
response=$(curl -k -X POST -u "$ES_USER:$ES_PASSWORD" -H "Content-Type: application/json" -d '{
75+
"name": "tmp-api-key",
76+
"expiration": "1d"
77+
}' "$DR_ELASTICSEARCH_URL/_security/api_key")
78+
79+
DR_API_KEY=$(echo "$response" | jq -r '.encoded')
80+
echo "::add-mask::$DR_API_KEY"
81+
echo "DR_API_KEY=$DR_API_KEY" >> $GITHUB_ENV
82+
4083
- name: Build release package with navigator files
84+
env:
85+
DR_REMOTE_ESQL_VALIDATION: "true"
86+
DR_CLOUD_ID: ${{ secrets.dr_cloud_id || '' }}
87+
DR_KIBANA_URL: ${{ secrets.dr_cloud_id == '' && 'https://localhost:5601' || '' }}
88+
DR_ELASTICSEARCH_URL: ${{ secrets.dr_cloud_id == '' && 'https://localhost:9200' || '' }}
89+
DR_API_KEY: ${{ secrets.dr_api_key || env.DR_API_KEY }}
90+
DR_IGNORE_SSL_ERRORS: ${{ secrets.dr_cloud_id == '' && 'true' || '' }}
4191
run: |
4292
python -m detection_rules dev build-release --generate-navigator
4393
@@ -56,6 +106,12 @@ jobs:
56106
- name: Lock the versions
57107
env:
58108
BRANCHES: "${{github.event.inputs.branches}}"
109+
DR_REMOTE_ESQL_VALIDATION: "true"
110+
DR_CLOUD_ID: ${{ secrets.dr_cloud_id || '' }}
111+
DR_KIBANA_URL: ${{ secrets.dr_cloud_id == '' && 'https://localhost:5601' || '' }}
112+
DR_ELASTICSEARCH_URL: ${{ secrets.dr_cloud_id == '' && 'https://localhost:9200' || '' }}
113+
DR_API_KEY: ${{ secrets.dr_api_key || env.DR_API_KEY }}
114+
DR_IGNORE_SSL_ERRORS: ${{ secrets.dr_cloud_id == '' && 'true' || '' }}
59115
run: |
60116
./detection_rules/etc/lock-multiple.sh $BRANCHES
61117
git add detection_rules/etc/version.lock.json

‎.github/workflows/pythonpackage.yml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,13 @@ jobs:
3737
env:
3838
# only run the test test_rule_change_has_updated_date on pull request events to main
3939
GITHUB_EVENT_NAME: "${{ github.event_name}}"
40+
# only run remote validation if repo is set to do so otherwise defer to .github/workflows/esql-validation.yml
41+
DR_REMOTE_ESQL_VALIDATION: "${{ vars.remote_esql_validation }}"
42+
DR_CLOUD_ID: ${{ secrets.dr_cloud_id }}
43+
DR_KIBANA_URL: ${{ secrets.dr_cloud_id }}
44+
DR_ELASTICSEARCH_URL: ${{ secrets.dr_cloud_id }}
45+
DR_API_KEY: ${{ secrets.dr_api_key }}
46+
DR_IGNORE_SSL_ERRORS: ${{ secrets.dr_cloud_id }}
4047
run: |
4148
python -m detection_rules test
4249

‎.github/workflows/release-fleet.yml‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,57 @@ jobs:
112112
git tag $RELEASE_TAG
113113
git push origin $RELEASE_TAG
114114
115+
- name: Check out container repository
116+
env:
117+
DR_CLOUD_ID: ${{ secrets.dr_cloud_id }}
118+
DR_API_KEY: ${{ secrets.dr_api_key }}
119+
if: ${{ !env.DR_CLOUD_ID && !env.DR_API_KEY }}
120+
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5
121+
with:
122+
path: elastic-container
123+
repository: peasead/elastic-container
124+
125+
- name: Build and run containers
126+
env:
127+
DR_CLOUD_ID: ${{ secrets.dr_cloud_id }}
128+
DR_API_KEY: ${{ secrets.dr_api_key }}
129+
if: ${{ !env.DR_CLOUD_ID && !env.DR_API_KEY }}
130+
run: |
131+
cd elastic-container
132+
GENERATED_PASSWORD=$(openssl rand -base64 16)
133+
sed -i "s|changeme|$GENERATED_PASSWORD|" .env
134+
echo "::add-mask::$GENERATED_PASSWORD"
135+
echo "GENERATED_PASSWORD=$GENERATED_PASSWORD" >> $GITHUB_ENV
136+
set -x
137+
bash elastic-container.sh start
138+
139+
- name: Get API Key and setup auth
140+
env:
141+
DR_CLOUD_ID: ${{ secrets.dr_cloud_id }}
142+
DR_API_KEY: ${{ secrets.dr_api_key }}
143+
DR_ELASTICSEARCH_URL: "https://localhost:9200"
144+
ES_USER: "elastic"
145+
ES_PASSWORD: ${{ env.GENERATED_PASSWORD }}
146+
if: ${{ !env.DR_CLOUD_ID && !env.DR_API_KEY }}
147+
run: |
148+
cd detection-rules
149+
response=$(curl -k -X POST -u "$ES_USER:$ES_PASSWORD" -H "Content-Type: application/json" -d '{
150+
"name": "tmp-api-key",
151+
"expiration": "1d"
152+
}' "$DR_ELASTICSEARCH_URL/_security/api_key")
153+
154+
DR_API_KEY=$(echo "$response" | jq -r '.encoded')
155+
echo "::add-mask::$DR_API_KEY"
156+
echo "DR_API_KEY=$DR_API_KEY" >> $GITHUB_ENV
157+
115158
- name: Build release package
159+
env:
160+
DR_REMOTE_ESQL_VALIDATION: "true"
161+
DR_CLOUD_ID: ${{ secrets.dr_cloud_id || '' }}
162+
DR_KIBANA_URL: ${{ secrets.dr_cloud_id == '' && 'https://localhost:5601' || '' }}
163+
DR_ELASTICSEARCH_URL: ${{ secrets.dr_cloud_id == '' && 'https://localhost:9200' || '' }}
164+
DR_API_KEY: ${{ secrets.dr_api_key || env.DR_API_KEY }}
165+
DR_IGNORE_SSL_ERRORS: ${{ secrets.dr_cloud_id == '' && 'true' || '' }}
116166
run: |
117167
cd detection-rules
118168
python -m detection_rules dev build-release

‎CLI.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,10 @@ Using the environment variable `DR_BYPASS_TIMELINE_TEMPLATE_VALIDATION` will byp
4949
Using the environment variable `DR_CLI_MAX_WIDTH` will set a custom max width for the click CLI.
5050
For instance, some users may want to increase the default value in cases where help messages are cut off.
5151

52+
Using the environment variable `DR_REMOTE_ESQL_VALIDATION` will enable remote ESQL validation for rules that use ESQL queries. This validation will be performed whenever the rule is loaded including for example the view-rule command. This requires the appropriate kibana_url or cloud_id, api_key, and es_url to be set in the config file or as environment variables.
53+
54+
Using the environment variable `DR_SKIP_EMPTY_INDEX_CLEANUP` will disable the cleanup of remote testing indexes that are created as part of the remote ESQL validation. By default, these indexes are deleted after the validation is complete, or upon validation error.
55+
5256
## Importing rules into the repo
5357

5458
You can import rules into the repo using the `create-rule` or `import-rules-to-repo` commands. Both of these commands will

0 commit comments

Comments
 (0)