Skip to content

Commit 42cf90d

Browse files
Merge branch 'main' into renovate_updates
2 parents 2ce702a + 64cc823 commit 42cf90d

File tree

88 files changed

+2575
-548
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

88 files changed

+2575
-548
lines changed

detection_rules/etc/non-ecs-schema.json

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -202,7 +202,8 @@
202202
"azure.activitylogs.properties.resourceDisplayName": "keyword",
203203
"azure.activitylogs.properties.appDisplayName": "keyword",
204204
"azure.activitylogs.properties.requestbody.properties.roleDefinitionId": "keyword",
205-
"azure.activitylogs.properties.responseBody": "keyword"
205+
"azure.activitylogs.properties.responseBody": "keyword",
206+
"azure.activitylogs.properties.status_code": "keyword"
206207
},
207208
"logs-azure.graphactivitylogs-*": {
208209
"azure.graphactivitylogs.properties.c_idtyp": "keyword",

detection_rules/index_mappings.py

Lines changed: 23 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -160,25 +160,33 @@ def get_simulated_index_template_mappings(elastic_client: Elasticsearch, name: s
160160

161161

162162
def prune_mappings_of_unsupported_types(
163-
integration: str, stream: str, stream_mappings: dict[str, Any], log: Callable[[str], None]
163+
debug_str_data_source: str, stream_mappings: dict[str, Any], log: Callable[[str], None]
164164
) -> dict[str, Any]:
165165
"""Prune fields with unsupported types (ES|QL) from the provided mappings."""
166166
nested_multifields = find_nested_multifields(stream_mappings)
167167
for field in nested_multifields:
168-
field_name = str(field).split(".fields.")[0].replace(".", ".properties.") + ".fields"
168+
parts = str(field).split(".fields.")[0].split(".")
169+
base_name = ".properties.".join(parts)
170+
field_name = f"{base_name}.fields"
169171
log(
170-
f"Warning: Nested multi-field `{field}` found in `{integration}-{stream}`. "
172+
f"Warning: Nested multi-field `{field}` found in `{debug_str_data_source}`. "
171173
f"Removing parent field from schema for ES|QL validation."
172174
)
173175
delete_nested_key_from_dict(stream_mappings, field_name)
174176
nested_flattened_fields = find_flattened_fields_with_subfields(stream_mappings)
175177
for field in nested_flattened_fields:
176-
field_name = str(field).split(".fields.")[0].replace(".", ".properties.") + ".fields"
178+
# Remove both .fields and .properties entries for flattened fields
179+
# .properties entries can occur when being merged with non-ecs or custom schemas
180+
parts = str(field).split(".fields.")[0].split(".")
181+
base_name = ".properties.".join(parts)
182+
field_name = f"{base_name}.fields"
183+
property_name = f"{base_name}.properties"
177184
log(
178-
f"Warning: flattened field `{field}` found in `{integration}-{stream}` with sub fields. "
185+
f"Warning: flattened field `{field}` found in `{debug_str_data_source}` with sub fields. "
179186
f"Removing parent field from schema for ES|QL validation."
180187
)
181188
delete_nested_key_from_dict(stream_mappings, field_name)
189+
delete_nested_key_from_dict(stream_mappings, property_name)
182190
return stream_mappings
183191

184192

@@ -222,7 +230,7 @@ def prepare_integration_mappings( # noqa: PLR0913
222230
for stream in package_schema:
223231
flat_schema = package_schema[stream]
224232
stream_mappings = flat_schema_to_index_mapping(flat_schema)
225-
stream_mappings = prune_mappings_of_unsupported_types(integration, stream, stream_mappings, log)
233+
stream_mappings = prune_mappings_of_unsupported_types(f"{integration}-{stream}", stream_mappings, log)
226234
utils.combine_dicts(integration_mappings, deepcopy(stream_mappings))
227235
index_lookup[f"{integration}-{stream}"] = stream_mappings
228236

@@ -246,12 +254,13 @@ def get_index_to_package_lookup(indices: list[str], index_lookup: dict[str, Any]
246254
return index_lookup_indices
247255

248256

249-
def get_filtered_index_schema(
257+
def get_filtered_index_schema( # noqa: PLR0913
250258
indices: list[str],
251259
index_lookup: dict[str, Any],
252260
ecs_schema: dict[str, Any],
253261
non_ecs_mapping: dict[str, Any],
254262
custom_mapping: dict[str, Any],
263+
log: Callable[[str], None],
255264
) -> tuple[dict[str, Any], dict[str, Any]]:
256265
"""Check if the provided indices are known based on the integration format. Returns the combined schema."""
257266

@@ -304,7 +313,7 @@ def get_filtered_index_schema(
304313
# Need to use a merge here to not overwrite existing fields
305314
utils.combine_dicts(base, deepcopy(non_ecs_mapping.get(match, {})))
306315
utils.combine_dicts(base, deepcopy(custom_mapping.get(match, {})))
307-
filtered_index_lookup[match] = base
316+
filtered_index_lookup[match] = prune_mappings_of_unsupported_types(match, base, log)
308317
utils.combine_dicts(combined_mappings, deepcopy(base))
309318

310319
# Reduce the index lookup to only the matched indices (remote/Kibana schema validation source of truth)
@@ -403,7 +412,7 @@ def find_nested_multifields(mapping: dict[str, Any], path: str = "") -> list[Any
403412

404413

405414
def find_flattened_fields_with_subfields(mapping: dict[str, Any], path: str = "") -> list[str]:
406-
"""Recursively search for fields of type 'flattened' that have a 'fields' key in Elasticsearch mappings."""
415+
"""Recursively search for type 'flattened' that have a 'fields' or 'properties' key in Elasticsearch mappings."""
407416
flattened_fields_with_subfields: list[str] = []
408417

409418
for field, properties in mapping.items():
@@ -413,6 +422,9 @@ def find_flattened_fields_with_subfields(mapping: dict[str, Any], path: str = ""
413422
# Check if the field is of type 'flattened' and has a 'fields' key
414423
if properties.get("type") == "flattened" and "fields" in properties: # type: ignore[reportUnknownVariableType]
415424
flattened_fields_with_subfields.append(current_path) # type: ignore[reportUnknownVariableType]
425+
# Check if the field is of type 'flattened' and has a 'properties' key
426+
if properties.get("type") == "flattened" and "properties" in properties: # type: ignore[reportUnknownVariableType]
427+
flattened_fields_with_subfields.append(current_path) # type: ignore[reportUnknownVariableType]
416428

417429
# Recurse into subfields
418430
if "properties" in properties:
@@ -487,8 +499,7 @@ def prepare_mappings( # noqa: PLR0913
487499
# and also at a per index level as custom schemas can override non-ecs fields and/or indices
488500
non_ecs_schema = ecs.flatten(non_ecs_schema)
489501
non_ecs_schema = utils.convert_to_nested_schema(non_ecs_schema)
490-
non_ecs_schema = prune_mappings_of_unsupported_types("non-ecs", "non-ecs", non_ecs_schema, log)
491-
non_ecs_mapping = prune_mappings_of_unsupported_types("non-ecs", "non-ecs", non_ecs_mapping, log)
502+
non_ecs_schema = prune_mappings_of_unsupported_types("non-ecs", non_ecs_schema, log)
492503

493504
# Load custom schema and convert to index mapping format (nested schema)
494505
custom_mapping: dict[str, Any] = {}
@@ -498,15 +509,14 @@ def prepare_mappings( # noqa: PLR0913
498509
index_mapping = ecs.flatten(index_mapping)
499510
index_mapping = utils.convert_to_nested_schema(index_mapping)
500511
custom_mapping.update({index: index_mapping})
501-
custom_mapping = prune_mappings_of_unsupported_types("custom", "custom", custom_mapping, log)
502512

503513
# Load ECS in an index mapping format (nested schema)
504514
current_version = Version.parse(load_current_package_version(), optional_minor_and_patch=True)
505515
ecs_schema = get_ecs_schema_mappings(current_version)
506516

507517
# Filter combined mappings based on the provided indices
508518
combined_mappings, index_lookup = get_filtered_index_schema(
509-
indices, index_lookup, ecs_schema, non_ecs_mapping, custom_mapping
519+
indices, index_lookup, ecs_schema, non_ecs_mapping, custom_mapping, log
510520
)
511521

512522
index_lookup.update({"rule-ecs-index": ecs_schema})
Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
# Windows Audit Policies
2+
3+
Windows related audit policies that need to be implemented in order to generate the events that power our detection rules. It serves as a centralized view of the policies we use so you don't need to go through every rule to know the different audit policies required.
4+
5+
Audit Policies:
6+
7+
* [Audit Authorization Policy Change](audit_authorization_policy_change.md)
8+
* [Audit Computer Account Management](audit_computer_account_management.md)
9+
* [Audit Detailed File Share](audit_detailed_file_share.md)
10+
* [Audit Directory Service Access](audit_directory_service_access.md)
11+
* [Audit Directory Service Changes](audit_directory_service_changes.md)
12+
* [Audit Filtering Platform Connection](audit_filtering_platform_connection.md)
13+
* [Audit Filtering Platform Packet Drop](audit_filtering_platform_packet_drop.md)
14+
* [Audit Handle Manipulation](audit_handle_manipulation.md)
15+
* [Audit Logon](audit_logon.md)
16+
* [Audit Other Object Access Events](audit_other_object_access_events.md)
17+
* [Audit Policy Change](audit_policy_change.md)
18+
* [Audit Process Creation and Command Line](audit_process_creation_and_command_line.md)
19+
* [Audit Security Group Management](audit_security_group_management.md)
20+
* [Audit Security System Extension](audit_security_system_extension.md)
21+
* [Audit Sensitive Privilege Use](audit_sensitive_privilege_use.md)
22+
* [Audit Special Logon](audit_special_logon.md)
23+
* [Audit Token Right Adjusted Events](audit_token_right_adjusted_events.md)
24+
* [Audit User Account Management](audit_user_account_management.md)
25+
* [Audit Powershell Script Block Logging](audit_powershell_scriptblock.md)
26+
27+
---
28+
29+
# Sysmon Configuration Guides
30+
31+
**Caution:** The following guides provide minimal configuration examples designed to enable specific Sysmon Event IDs. Collecting Sysmon events without a tailored configuration for your environment will cause high data volume and potentially high CPU-load, and these setup instructions require significant tuning to be production-ready.
32+
33+
To build an efficient and production-ready configuration, we strongly recommend exploring these community resources:
34+
35+
- [TrustedSec Sysmon Community Guide](https://github.com/trustedsec/SysmonCommunityGuide)
36+
- [olafhartong - sysmon-modular](https://github.com/olafhartong/sysmon-modular)
37+
- [Neo23x0 - sysmon-config](https://github.com/Neo23x0/sysmon-config)
38+
39+
For a production-ready and more integrated solution that is designed to work with our detection rules and also provide native Endpoint Protection and Response, check out [Elastic Endpoint Security](https://www.elastic.co/security/endpoint-security).
40+
41+
* [Sysmon Event ID 1: Process Creation](sysmon_eventid1_process_creation.md)
42+
* [Sysmon Event ID 2: File Creation Time Changed](sysmon_eventid2_file_creation_time_changed.md)
43+
* [Sysmon Event ID 3: Network Connection](sysmon_eventid3_network_connection.md)
44+
* [Sysmon Event ID 7: Image Loaded](sysmon_eventid7_image_loaded.md)
45+
* [Sysmon Event ID 8: Create Remote Thread](sysmon_eventid8_createremotethread.md)
46+
* [Sysmon Event ID 10: Process Accessed](sysmon_eventid10_process_access.md)
47+
* [Sysmon Event ID 11: File Create](sysmon_eventid11_file_create.md)
48+
* [Sysmon Event IDs 12, 13, 14: Registry Events](sysmon_eventid12_13_14_registry_event.md)
49+
* [Sysmon Event IDs 17, 18: Named Pipe Events](sysmon_eventid17_18_pipe_event.md)
50+
* [Sysmon Event IDs 19, 20, 21: WMI Events](sysmon_eventid19_20_21_wmi_event.md)
51+
* [Sysmon Event ID 22: DNS Query](sysmon_eventid22_dns_query.md)
52+
* [Sysmon Event ID 23: File Delete](sysmon_eventid23_file_delete.md)
Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
# Audit Authorization Policy Change
2+
3+
## Setup
4+
5+
Some detection rules require monitoring changes to authorization policies to detect unauthorized modifications or misconfigurations. Enabling this setting ensures visibility into changes affecting user rights and security policies, helping maintain compliance and security.
6+
7+
**Caution:** Enabling this audit policy can generate a high volume of events. Evaluate the audit policy in a group of servers to measure volume and filter unwanted events before deploying in the entire domain.
8+
9+
### Enable Audit Policy via Group Policy
10+
11+
To enable `Audit Authorization Policy Change` across a group of servers using Active Directory Group Policies, administrators must enable the `Audit Authorization Policy Change` policy. Follow these steps to configure the audit policy via Advanced Audit Policy Configuration:
12+
13+
```
14+
Computer Configuration >
15+
Windows Settings >
16+
Security Settings >
17+
Advanced Audit Policy Configuration >
18+
Audit Policies >
19+
Policy Change >
20+
Audit Authorization Policy Change (Success,Failure)
21+
```
22+
23+
### Enable Locally using auditpol
24+
25+
To enable this policy on a local machine, run the following command in an elevated command prompt:
26+
27+
```
28+
auditpol.exe /set /subcategory:"Authorization Policy Change" /success:enable /failure:enable
29+
```
30+
31+
## Event IDs
32+
33+
When this audit policy is enabled, the following event IDs may be generated:
34+
35+
* **4703**: A user right was adjusted.
36+
* **4704**: A user right was assigned.
37+
* **4705**: A user right was removed.
38+
* **4670**: Permissions on an object were changed.
39+
* **4911**: Resource attributes of the object were changed.
40+
* **4913**: Central Access Policy on the object was changed.
41+
42+
## Related Rules
43+
44+
Use the following GitHub search to identify rules that use the events listed:
45+
46+
[Elastic Detection Rules Github Repo Search](https://github.com/search?q=repo%3Aelastic%2Fdetection-rules+%22Windows+Security+Event+Logs%22+AND+%28%224703%22+OR+%22Token+Right+Adjusted+Events%22+OR+%224704%22+OR+%22user-right-assigned%22+OR+%224705%22+OR+%22user-right-removed%22+OR+%224670%22+OR+%22permissions-changed%22+OR+%224911%22+OR+%224913%22%29++language%3ATOML&type=code)
Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
# Audit Computer Account Management
2+
3+
## Setup
4+
5+
Some detection rules require monitoring computer account management events to track changes to computer accounts in the domain. Enabling this setting provides visibility into when computer accounts are created, changed, or deleted, which is crucial for detecting potential malicious activity like adding unauthorized computer accounts.
6+
7+
### Enable Audit Policy via Group Policy
8+
9+
To enable `Audit Computer Account Management` events across a group of servers using Active Directory Group Policies, administrators must enable the `Audit Computer Account Management` policy. Follow these steps to configure the audit policy via Advanced Audit Policy Configuration:
10+
11+
```
12+
Computer Configuration >
13+
Policies >
14+
Windows Settings >
15+
Security Settings >
16+
Advanced Audit Policies Configuration >
17+
Audit Policies >
18+
Account Management >
19+
Audit Computer Account Management (Success,Failure)
20+
```
21+
22+
### Enable Locally using auditpol
23+
24+
To enable this policy on a local machine, run the following command in an elevated command prompt:
25+
26+
```
27+
auditpol.exe /set /subcategory:"Computer Account Management" /success:enable /failure:enable
28+
```
29+
30+
## Event IDs
31+
32+
When this audit policy is enabled, the following event IDs may be generated:
33+
34+
* **4741**: A computer account was created.
35+
* **4742**: A computer account was changed.
36+
* **4743**: A computer account was deleted.
37+
38+
## Related Rules
39+
40+
Use the following GitHub search to identify rules that use the events listed:
41+
42+
[Elastic Detection Rules Github Repo Search](https://github.com/search?q=repo%3Aelastic%2Fdetection-rules+%22Windows+Security+Event+Logs%22+AND+%28%224741%22+OR+%22added-computer-account%22+OR+%224742%22+OR+%22changed-computer-account%22+OR+%224743%22+OR+%22deleted-computer-account%22%29+language%3ATOML+AND+NOT+%28%22%28for+example%2C+4741%29%22+OR+%22Review+the+event+ID+4741%22+OR+%22e.g.%2C+4741%22%29&type=code)
Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
# Audit Detailed File Share
2+
3+
## Setup
4+
5+
Some detection rules require monitoring file share access to detect unauthorized access attempts or modifications. Enabling this setting helps improve security visibility and ensures compliance by tracking access to shared files and folders.
6+
7+
**Caution:** Enabling this audit policy can generate a high volume of events. Evaluate the audit policy in a group of servers to measure volume and filter unwanted events before deploying in the entire domain.
8+
9+
### Enable Audit Policy via Group Policy
10+
11+
To enable `Audit Detailed File Share` across a group of servers using Active Directory Group Policies, administrators must enable the `Audit Detailed File Share` policy. Follow these steps to configure the audit policy via Advanced Audit Policy Configuration:
12+
13+
```
14+
Computer Configuration >
15+
Policies >
16+
Windows Settings >
17+
Security Settings >
18+
Advanced Audit Policies Configuration >
19+
Audit Policies >
20+
Object Access >
21+
Audit Detailed File Share (Success,Failure)
22+
```
23+
24+
### Enable Locally using auditpol
25+
26+
To enable this policy on a local machine, run the following command in an elevated command prompt:
27+
28+
```
29+
auditpol.exe /set /subcategory:"File Share" /success:enable /failure:disable
30+
```
31+
32+
## Event IDs
33+
34+
When this audit policy is enabled, the following event IDs may be generated:
35+
36+
* **5145**: A network share object was checked to see whether client can be granted desired access.
37+
38+
## Related Rules
39+
40+
Use the following GitHub search to identify rules that use the events listed:
41+
42+
[Elastic Detection Rules Github Repo Search](https://github.com/search?q=repo%3Aelastic%2Fdetection-rules+%22Windows+Security+Event+Logs%22+AND+%28%225145%22+OR+%22network-share-object-access-checked%22%29++language%3ATOML&type=code)
Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
# Audit Directory Service Access
2+
3+
## Setup
4+
5+
Some detection rules require configuring audit policies to generate events when Active Directory objects are accessed. These audit policies apply exclusively to Domain Controllers, as other servers do not produce events related to Active Directory object modifications.
6+
7+
**Caution:** Enabling this audit policy can generate a high volume of events. Evaluate the audit policy in a group of servers to measure volume and filter unwanted events before deploying in the entire domain.
8+
9+
### Enable Audit Policy via Group Policy
10+
11+
To enable `Audit Directory Service Access` on all Domain Controllers via Group Policy, administrators must enable the `Audit Directory Service Access` policy. Follow these steps to configure the audit policy via Advanced Audit Policy Configuration:
12+
13+
```
14+
Computer Configuration >
15+
Policies >
16+
Windows Settings >
17+
Security Settings >
18+
Advanced Audit Policies Configuration >
19+
Audit Policies >
20+
DS Access >
21+
Audit Directory Service Access (Success,Failure)
22+
```
23+
24+
### Enable Locally using auditpol
25+
26+
To enable this policy on a local machine, run the following command in an elevated command prompt:
27+
28+
```
29+
auditpol.exe /set /subcategory:"Directory Service Access" /success:enable /failure:enable
30+
```
31+
32+
## Event IDs
33+
34+
When this audit policy is enabled, the following event IDs may be generated:
35+
36+
* **4661**: A handle to an object was requested.
37+
* **4662**: An operation was performed on an object.
38+
39+
## Related Rules
40+
41+
Use the following GitHub search to identify rules that use the events listed:
42+
43+
[Elastic Detection Rules Github Repo Search](https://github.com/search?q=repo%3Aelastic%2Fdetection-rules+%22Windows+Security+Event+Logs%22+AND+%28%224661%22+OR+%224662%22+OR+%22object-operation-performed%22%29++language%3ATOML&type=code)

0 commit comments

Comments
 (0)