fix: Skip invalid YAML files in Beats dist (#4865)

* Skip invalid YAML files but keep them in the branch

* Typo fix

* Patch version bump

* Adding a schema generation command to `test_cli.bash` flow

Author: traut

detection_rules/beats.py

@@ -21,7 +21,7 @@
 
 
 def _decompress_and_save_schema(url: str, release_name: str) -> None:
-    print(f"Downloading beats {release_name}")
+    print(f"Downloading beats {release_name}", url)
     response = requests.get(url, timeout=30)
 
     print(f"Downloaded {len(response.content) / 1024.0 / 1024.0:.2f} MB release.")
@@ -34,26 +34,25 @@ def _decompress_and_save_schema(url: str, release_name: str) -> None:
         for name in archive.namelist():
             path = Path(name)
             if path.name in ("fields.yml", "fields.common.yml", "config.yml"):
-                contents = archive.read(name)
-
                 # chop off the base directory name
                 key = name[len(base_directory) :]
 
                 if key.startswith("x-pack"):
                     key = key[len("x-pack") + 1 :]
 
-                try:
-                    decoded = yaml.safe_load(contents)
-                except yaml.YAMLError as e:
-                    print(f"Error loading {name}")
-                    raise ValueError(f"Error loading {name}") from e
-
                 # create a hierarchical structure
                 branch = fs
                 directory, base_name = os.path.split(key)
                 for limb in directory.split(os.path.sep):
                     branch = branch.setdefault("folders", {}).setdefault(limb, {})
 
+                contents = archive.read(name)
+                try:
+                    decoded = yaml.safe_load(contents)
+                except yaml.YAMLError:
+                    print(f"Error loading {name}, not a valid YAML")
+                    decoded = None
+
                 branch.setdefault("files", {})[base_name] = decoded
 
     # remove all non-beat directories

detection_rules/etc/test_cli.bash

@@ -25,6 +25,9 @@ rm -rf tmp-export
 echo "Updating rule data schemas"
 python -m detection_rules dev schemas update-rule-data
 
+echo "Generate Beats schemas"
+GITHUB_TOKEN="foo" python -m detection_rules dev schemas generate --schema beats
+
 echo "Validating rule: execution_github_new_event_action_for_pat.toml"
 python -m detection_rules validate-rule rules_building_block/execution_github_new_event_action_for_pat.toml
 

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "detection_rules"
-version = "1.3.0"
+version = "1.3.1"
 description = "Detection Rules is the home for rules used by Elastic Security. This repository is used for the development, maintenance, testing, validation, and release of rules for Elastic Security’s Detection Engine."
 readme = "README.md"
 requires-python = ">=3.12"