From cdb346cb77c1cc44d6bbbf924f9ad999815ac1c1 Mon Sep 17 00:00:00 2001
From: Sergey Polzunov <traut@users.noreply.github.com>
Date: Wed, 2 Jul 2025 13:39:35 +0200
Subject: [PATCH] fix: Skip invalid YAML files in Beats dist (#4865)

* Skip invalid YAML files but keep them in the branch

* Typo fix

* Patch version bump

* Adding a schema generation command to `test_cli.bash` flow
---
 detection_rules/beats.py          | 17 ++++++++---------
 detection_rules/etc/test_cli.bash |  3 +++
 pyproject.toml                    |  2 +-
 3 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/detection_rules/beats.py b/detection_rules/beats.py
index d5afaa923..094bf358c 100644
--- a/detection_rules/beats.py
+++ b/detection_rules/beats.py
@@ -21,7 +21,7 @@ from .utils import DateTimeEncoder, cached, get_etc_path, gzip_compress, read_gz
 
 
 def _decompress_and_save_schema(url: str, release_name: str) -> None:
-    print(f"Downloading beats {release_name}")
+    print(f"Downloading beats {release_name}", url)
     response = requests.get(url, timeout=30)
 
     print(f"Downloaded {len(response.content) / 1024.0 / 1024.0:.2f} MB release.")
@@ -34,26 +34,25 @@ def _decompress_and_save_schema(url: str, release_name: str) -> None:
         for name in archive.namelist():
             path = Path(name)
             if path.name in ("fields.yml", "fields.common.yml", "config.yml"):
-                contents = archive.read(name)
-
                 # chop off the base directory name
                 key = name[len(base_directory) :]
 
                 if key.startswith("x-pack"):
                     key = key[len("x-pack") + 1 :]
 
-                try:
-                    decoded = yaml.safe_load(contents)
-                except yaml.YAMLError as e:
-                    print(f"Error loading {name}")
-                    raise ValueError(f"Error loading {name}") from e
-
                 # create a hierarchical structure
                 branch = fs
                 directory, base_name = os.path.split(key)
                 for limb in directory.split(os.path.sep):
                     branch = branch.setdefault("folders", {}).setdefault(limb, {})
 
+                contents = archive.read(name)
+                try:
+                    decoded = yaml.safe_load(contents)
+                except yaml.YAMLError:
+                    print(f"Error loading {name}, not a valid YAML")
+                    decoded = None
+
                 branch.setdefault("files", {})[base_name] = decoded
 
     # remove all non-beat directories
diff --git a/detection_rules/etc/test_cli.bash b/detection_rules/etc/test_cli.bash
index d3ef7091f..9cc31e0e1 100755
--- a/detection_rules/etc/test_cli.bash
+++ b/detection_rules/etc/test_cli.bash
@@ -25,6 +25,9 @@ rm -rf tmp-export
 echo "Updating rule data schemas"
 python -m detection_rules dev schemas update-rule-data
 
+echo "Generate Beats schemas"
+GITHUB_TOKEN="foo" python -m detection_rules dev schemas generate --schema beats
+
 echo "Validating rule: execution_github_new_event_action_for_pat.toml"
 python -m detection_rules validate-rule rules_building_block/execution_github_new_event_action_for_pat.toml
 
diff --git a/pyproject.toml b/pyproject.toml
index f87ed96d5..793857570 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "detection_rules"
-version = "1.3.0"
+version = "1.3.1"
 description = "Detection Rules is the home for rules used by Elastic Security. This repository is used for the development, maintenance, testing, validation, and release of rules for Elastic Security’s Detection Engine."
 readme = "README.md"
 requires-python = ">=3.12"