Add testcase to check for related_integrations based on index (#4096)

shashank-elastic · github-actions[bot] · commit aacea42d1d85 · 2024-10-21T18:50:02.000Z
Removed changes from: - rules/windows/command_and_control_rdp_tunnel_plink.toml - rules/windows/command_and_control_screenconnect_childproc.toml - rules/windows/command_and_control_tunnel_vscode.toml - rules/windows/credential_access_cmdline_dump_tool.toml - rules/windows/credential_access_persistence_network_logon_provider_modification.toml - rules/windows/credential_access_saved_creds_vaultcmd.toml - rules/windows/credential_access_via_snapshot_lsass_clone_creation.toml - rules/windows/defense_evasion_defender_disabled_via_registry.toml - rules/windows/defense_evasion_from_unusual_directory.toml - rules/windows/defense_evasion_sip_provider_mod.toml - rules/windows/defense_evasion_suspicious_zoom_child_process.toml - rules/windows/defense_evasion_unusual_system_vp_child_program.toml - rules/windows/defense_evasion_via_filter_manager.toml - rules/windows/execution_com_object_xwizard.toml - rules/windows/execution_suspicious_pdf_reader.toml - rules/windows/execution_via_mmc_console_file_unusual_path.toml - rules/windows/impact_volume_shadow_copy_deletion_via_wmic.toml - rules/windows/lateral_movement_execution_from_tsclient_mup.toml - rules/windows/lateral_movement_unusual_dns_service_children.toml - rules/windows/persistence_registry_uncommon.toml - rules/windows/persistence_via_update_orchestrator_service_hijack.toml - rules/windows/privilege_escalation_dns_serverlevelplugindll.toml - rules/windows/privilege_escalation_msi_repair_via_mshelp_link.toml - rules/windows/privilege_escalation_printspooler_registry_copyfiles.toml (selectively cherry picked from commit 275c728)
diff --git a/detection_rules/schemas/definitions.py b/detection_rules/schemas/definitions.py
@@ -247,3 +247,11 @@ def validator(value):
     'geo_point', 'geo_shape', 'point', 'shape',
     'percolator'
 ]
+
+# definitions for the integration to index mapping unit test case
+IGNORE_IDS = ["eb079c62-4481-4d6e-9643-3ca499df7aaa", "699e9fdb-b77c-4c01-995c-1c15019b9c43",
+              "0c9a14d9-d65d-486f-9b5b-91e4e6b22bd0", "a198fbbd-9413-45ec-a269-47ae4ccf59ce",
+              "0c41e478-5263-4c69-8f9e-7dfd2c22da64", "aab184d3-72b3-4639-b242-6597c99d8bca",
+              "a61809f3-fb5b-465c-8bff-23a8a068ac60", "f3e22c8b-ea47-45d1-b502-b57b6de950b3"]
+IGNORE_INDICES = ['.alerts-security.*', 'logs-*', 'metrics-*', 'traces-*', 'endgame-*',
+                  'filebeat-*', 'packetbeat-*', 'auditbeat-*', 'winlogbeat-*']
diff --git a/rules/cross-platform/execution_aws_ssm_sendcommand_with_command_parameters.toml b/rules/cross-platform/execution_aws_ssm_sendcommand_with_command_parameters.toml
@@ -1,8 +1,8 @@
 [metadata]
 creation_date = "2022/09/03"
-integration = ["endpoint"]
+integration = ["endpoint", "auditd_manager"]
 maturity = "production"
-updated_date = "2024/09/10"
+updated_date = "2024/09/23"
 
 [rule]
 author = ["Elastic"]
diff --git a/rules/windows/command_and_control_new_terms_commonly_abused_rat_execution.toml b/rules/windows/command_and_control_new_terms_commonly_abused_rat_execution.toml
@@ -1,8 +1,8 @@
 [metadata]
 creation_date = "2023/04/03"
-integration = ["endpoint", "windows"]
+integration = ["endpoint", "windows", "system"]
 maturity = "production"
-updated_date = "2024/08/07"
+updated_date = "2024/09/23"
 
 [rule]
 author = ["Elastic"]
diff --git a/rules/windows/credential_access_dcsync_newterm_subjectuser.toml b/rules/windows/credential_access_dcsync_newterm_subjectuser.toml
@@ -1,8 +1,8 @@
 [metadata]
 creation_date = "2022/12/19"
-integration = ["windows"]
+integration = ["windows", "system"]
 maturity = "production"
-updated_date = "2024/08/07"
+updated_date = "2024/09/23"
 
 [rule]
 author = ["Elastic"]
diff --git a/rules/windows/defense_evasion_cve_2020_0601.toml b/rules/windows/defense_evasion_cve_2020_0601.toml
@@ -1,8 +1,8 @@
 [metadata]
 creation_date = "2020/03/19"
-integration = ["windows"]
+integration = ["windows", "system"]
 maturity = "production"
-updated_date = "2024/08/07"
+updated_date = "2024/09/23"
 
 [rule]
 author = ["Elastic"]
diff --git a/rules/windows/impact_stop_process_service_threshold.toml b/rules/windows/impact_stop_process_service_threshold.toml
@@ -1,8 +1,8 @@
 [metadata]
 creation_date = "2020/12/03"
-integration = ["endpoint", "windows"]
+integration = ["endpoint", "windows", "system"]
 maturity = "production"
-updated_date = "2024/08/07"
+updated_date = "2024/09/23"
 
 [rule]
 author = ["Elastic"]
diff --git a/rules_building_block/collection_files_staged_in_recycle_bin_root.toml b/rules_building_block/collection_files_staged_in_recycle_bin_root.toml
@@ -1,8 +1,8 @@
 [metadata]
 creation_date = "2023/08/24"
-integration = ["endpoint"]
+integration = ["endpoint", "windows"]
 maturity = "production"
-updated_date = "2024/08/07"
+updated_date = "2024/09/23"
 
 [rule]
 author = ["Elastic"]
diff --git a/rules_building_block/collection_outlook_email_archive.toml b/rules_building_block/collection_outlook_email_archive.toml
@@ -1,8 +1,8 @@
 [metadata]
 creation_date = "2023/08/21"
-integration = ["endpoint"]
+integration = ["endpoint", "windows", "system"]
 maturity = "production"
-updated_date = "2024/08/09"
+updated_date = "2024/09/23"
 
 [rule]
 author = ["Elastic"]
diff --git a/rules_building_block/command_and_control_bitsadmin_activity.toml b/rules_building_block/command_and_control_bitsadmin_activity.toml
@@ -1,8 +1,8 @@
 [metadata]
 creation_date = "2023/08/21"
-integration = ["endpoint"]
+integration = ["endpoint", "windows", "system"]
 maturity = "production"
-updated_date = "2024/08/09"
+updated_date = "2024/09/23"
 
 [rule]
 author = ["Elastic"]
diff --git a/rules_building_block/credential_access_win_private_key_access.toml b/rules_building_block/credential_access_win_private_key_access.toml
@@ -1,8 +1,8 @@
 [metadata]
 creation_date = "2023/08/21"
-integration = ["endpoint"]
+integration = ["endpoint", "windows", "system"]
 maturity = "production"
-updated_date = "2024/08/09"
+updated_date = "2024/09/23"
 
 [rule]
 author = ["Elastic"]
diff --git a/rules_building_block/defense_evasion_cmd_copy_binary_contents.toml b/rules_building_block/defense_evasion_cmd_copy_binary_contents.toml
@@ -1,8 +1,8 @@
 [metadata]
 creation_date = "2023/08/23"
-integration = ["endpoint"]
+integration = ["endpoint", "windows", "system"]
 maturity = "production"
-updated_date = "2024/08/09"
+updated_date = "2024/09/23"
 
 [rule]
 author = ["Elastic"]
diff --git a/rules_building_block/defense_evasion_cmstp_execution.toml b/rules_building_block/defense_evasion_cmstp_execution.toml
@@ -1,8 +1,8 @@
 [metadata]
 creation_date = "2023/08/24"
-integration = ["endpoint"]
+integration = ["endpoint", "windows", "system"]
 maturity = "production"
-updated_date = "2024/08/09"
+updated_date = "2024/09/23"
 
 [rule]
 author = ["Elastic"]
diff --git a/rules_building_block/defense_evasion_indirect_command_exec_pcalua_forfiles.toml b/rules_building_block/defense_evasion_indirect_command_exec_pcalua_forfiles.toml
@@ -1,8 +1,8 @@
 [metadata]
 creation_date = "2023/08/24"
-integration = ["endpoint"]
+integration = ["endpoint", "windows", "system"]
 maturity = "production"
-updated_date = "2024/08/07"
+updated_date = "2024/09/23"
 
 [rule]
 author = ["Elastic"]
diff --git a/rules_building_block/defense_evasion_installutil_command_activity.toml b/rules_building_block/defense_evasion_installutil_command_activity.toml
@@ -1,8 +1,8 @@
 [metadata]
 creation_date = "2023/08/24"
-integration = ["endpoint"]
+integration = ["endpoint", "windows", "system"]
 maturity = "production"
-updated_date = "2024/08/07"
+updated_date = "2024/09/23"
 
 [rule]
 author = ["Elastic"]
diff --git a/rules_building_block/defense_evasion_msdt_suspicious_diagcab.toml b/rules_building_block/defense_evasion_msdt_suspicious_diagcab.toml
@@ -1,8 +1,8 @@
 [metadata]
 creation_date = "2023/09/26"
-integration = ["endpoint"]
+integration = ["endpoint", "windows", "system"]
 maturity = "production"
-updated_date = "2024/08/07"
+updated_date = "2024/09/23"
 
 [rule]
 author = ["Elastic"]
diff --git a/rules_building_block/defense_evasion_service_path_registry.toml b/rules_building_block/defense_evasion_service_path_registry.toml
@@ -1,8 +1,8 @@
 [metadata]
 creation_date = "2023/08/29"
-integration = ["endpoint"]
+integration = ["endpoint", "windows"]
 maturity = "production"
-updated_date = "2024/08/07"
+updated_date = "2024/09/23"
 
 [rule]
 author = ["Elastic"]
diff --git a/rules_building_block/defense_evasion_services_exe_path.toml b/rules_building_block/defense_evasion_services_exe_path.toml
@@ -1,8 +1,8 @@
 [metadata]
 creation_date = "2023/08/29"
-integration = ["endpoint"]
+integration = ["endpoint", "windows", "system"]
 maturity = "production"
-updated_date = "2024/08/09"
+updated_date = "2024/09/23"
 
 [rule]
 author = ["Elastic"]
diff --git a/rules_building_block/defense_evasion_suspicious_msiexec_execution.toml b/rules_building_block/defense_evasion_suspicious_msiexec_execution.toml
@@ -1,8 +1,8 @@
 [metadata]
 creation_date = "2023/09/26"
-integration = ["endpoint"]
+integration = ["endpoint", "windows"]
 maturity = "production"
-updated_date = "2024/05/21"
+updated_date = "2024/09/23"
 
 [rule]
 author = ["Elastic"]
diff --git a/rules_building_block/defense_evasion_unusual_process_path_wbem.toml b/rules_building_block/defense_evasion_unusual_process_path_wbem.toml
@@ -1,8 +1,8 @@
 [metadata]
 creation_date = "2023/08/23"
-integration = ["endpoint"]
+integration = ["endpoint", "windows", "system"]
 maturity = "production"
-updated_date = "2024/08/07"
+updated_date = "2024/09/23"
 
 [rule]
 author = ["Elastic"]
diff --git a/rules_building_block/discovery_files_dir_systeminfo_via_cmd.toml b/rules_building_block/discovery_files_dir_systeminfo_via_cmd.toml
@@ -1,9 +1,9 @@
 [metadata]
 bypass_bbr_timing = true
 creation_date = "2022/11/01"
-integration = ["endpoint", "windows"]
+integration = ["endpoint", "windows", "system"]
 maturity = "production"
-updated_date = "2024/08/07"
+updated_date = "2024/09/23"
 
 [rule]
 author = ["Elastic"]
diff --git a/rules_building_block/discovery_generic_process_discovery.toml b/rules_building_block/discovery_generic_process_discovery.toml
@@ -1,9 +1,9 @@
 [metadata]
 bypass_bbr_timing = true
 creation_date = "2023/07/13"
-integration = ["endpoint"]
+integration = ["endpoint", "windows", "system"]
 maturity = "production"
-updated_date = "2024/08/07"
+updated_date = "2024/09/23"
 
 [rule]
 author = ["Elastic"]
diff --git a/rules_building_block/discovery_security_software_wmic.toml b/rules_building_block/discovery_security_software_wmic.toml
@@ -1,9 +1,9 @@
 [metadata]
 bypass_bbr_timing = true
 creation_date = "2020/10/19"
-integration = ["endpoint", "windows"]
+integration = ["endpoint", "windows", "system"]
 maturity = "production"
-updated_date = "2024/08/07"
+updated_date = "2024/09/23"
 
 [rule]
 author = ["Elastic"]
diff --git a/rules_building_block/discovery_system_service_discovery.toml b/rules_building_block/discovery_system_service_discovery.toml
@@ -1,9 +1,9 @@
 [metadata]
 bypass_bbr_timing = true
 creation_date = "2023/01/24"
-integration = ["windows", "endpoint"]
+integration = ["windows", "endpoint", "system"]
 maturity = "production"
-updated_date = "2024/08/07"
+updated_date = "2024/09/23"
 
 [rule]
 author = ["Elastic"]
diff --git a/rules_building_block/discovery_system_time_discovery.toml b/rules_building_block/discovery_system_time_discovery.toml
@@ -1,9 +1,9 @@
 [metadata]
 bypass_bbr_timing = true
 creation_date = "2023/01/24"
-integration = ["windows", "endpoint"]
+integration = ["windows", "endpoint", "system"]
 maturity = "production"
-updated_date = "2024/08/07"
+updated_date = "2024/09/23"
 
 [rule]
 author = ["Elastic"]
diff --git a/rules_building_block/discovery_windows_system_information_discovery.toml b/rules_building_block/discovery_windows_system_information_discovery.toml
@@ -1,9 +1,9 @@
 [metadata]
 bypass_bbr_timing = true
 creation_date = "2023/07/06"
-integration = ["windows", "endpoint"]
+integration = ["windows", "endpoint", "system"]
 maturity = "production"
-updated_date = "2024/08/07"
+updated_date = "2024/09/23"
 
 [rule]
 author = ["Elastic"]
diff --git a/rules_building_block/execution_settingcontent_ms_file_creation.toml b/rules_building_block/execution_settingcontent_ms_file_creation.toml
@@ -1,9 +1,9 @@
 [metadata]
 bypass_bbr_timing = true
 creation_date = "2023/08/24"
-integration = ["endpoint"]
+integration = ["endpoint", "windows"]
 maturity = "production"
-updated_date = "2024/08/07"
+updated_date = "2024/09/23"
 
 [rule]
 author = ["Elastic"]
diff --git a/rules_building_block/execution_wmi_wbemtest.toml b/rules_building_block/execution_wmi_wbemtest.toml
@@ -1,8 +1,8 @@
 [metadata]
 creation_date = "2023/08/24"
-integration = ["endpoint"]
+integration = ["endpoint", "windows", "system"]
 maturity = "production"
-updated_date = "2024/08/07"
+updated_date = "2024/09/23"
 
 [rule]
 author = ["Elastic"]
diff --git a/rules_building_block/lateral_movement_at.toml b/rules_building_block/lateral_movement_at.toml
@@ -1,8 +1,8 @@
 [metadata]
 creation_date = "2023/08/21"
-integration = ["endpoint"]
+integration = ["endpoint", "windows", "system"]
 maturity = "production"
-updated_date = "2024/08/07"
+updated_date = "2024/09/23"
 
 [rule]
 author = ["Elastic"]
diff --git a/rules_building_block/lateral_movement_wmic_remote.toml b/rules_building_block/lateral_movement_wmic_remote.toml
@@ -1,8 +1,8 @@
 [metadata]
 creation_date = "2023/08/24"
-integration = ["endpoint"]
+integration = ["endpoint", "windows", "system"]
 maturity = "production"
-updated_date = "2024/08/09"
+updated_date = "2024/09/23"
 
 [rule]
 author = ["Elastic"]
diff --git a/tests/test_all_rules.py b/tests/test_all_rules.py
@@ -791,25 +791,40 @@ def test_integration_tag(self):
                 else:
                     # checks if rule has index pattern integration and the integration tag exists
                     # ignore the External Alerts rule, Threat Indicator Matching Rules, Guided onboarding
-                    ignore_ids = [
-                        "eb079c62-4481-4d6e-9643-3ca499df7aaa",
-                        "699e9fdb-b77c-4c01-995c-1c15019b9c43",
-                        "0c9a14d9-d65d-486f-9b5b-91e4e6b22bd0",
-                        "a198fbbd-9413-45ec-a269-47ae4ccf59ce",
-                        "0c41e478-5263-4c69-8f9e-7dfd2c22da64",
-                        "aab184d3-72b3-4639-b242-6597c99d8bca",
-                        "a61809f3-fb5b-465c-8bff-23a8a068ac60",
-                        "f3e22c8b-ea47-45d1-b502-b57b6de950b3"
-                    ]
                     if any([re.search("|".join(non_dataset_packages), i, re.IGNORECASE)
                             for i in rule.contents.data.get('index') or []]):
-                        if not rule.contents.metadata.integration and rule.id not in ignore_ids and \
+                        if not rule.contents.metadata.integration and rule.id not in definitions.IGNORE_IDS and \
                                 rule.contents.data.type not in definitions.MACHINE_LEARNING:
                             err_msg = f'substrings {non_dataset_packages} found in '\
                                       f'{self.rule_str(rule)} rule index patterns are {rule.contents.data.index},' \
                                       f'but no integration tag found'
                             failures.append(err_msg)
 
+                # checks for a defined index pattern, the related integration exists in metadata
+                expected_integrations, missing_integrations = set(), set()
+                ignore_ml_packages = any(ri in [*map(str.lower, definitions.MACHINE_LEARNING_PACKAGES)]
+                                         for ri in rule_integrations)
+                for index in indices:
+                    if index in definitions.IGNORE_INDICES or ignore_ml_packages or \
+                            rule.id in definitions.IGNORE_IDS or rule.contents.data.type == 'threat_match':
+                        continue
+                    # Outlier integration log pattern to identify integration
+                    if index == 'apm-*-transaction*':
+                        index_map = ['apm']
+                    else:
+                        # Split by hyphen to get the second part of index
+                        index_part1, _, index_part2 = index.partition('-')
+                        #  Use regular expression to extract alphanumeric words, which is integration name
+                        parsed_integration = re.search(r'\b\w+\b', index_part2 or index_part1)
+                        index_map = [parsed_integration.group(0) if parsed_integration else None]
+                    if not index_map:
+                        self.fail(f'{self.rule_str(rule)} Could not determine the integration from Index {index}')
+                    expected_integrations.update(index_map)
+                missing_integrations.update(expected_integrations.difference(set(rule_integrations)))
+                if missing_integrations:
+                    error_msg = f'{self.rule_str(rule)} Missing integration metadata: {", ".join(missing_integrations)}'
+                    failures.append(error_msg)
+
         if failures:
             err_msg = """
                 The following rules have missing or invalid integrations tags.
