From 9736407ef33522780e11d69587336fb9d5bdc232 Mon Sep 17 00:00:00 2001 From: Eric Forte <119343520+eric-forte-elastic@users.noreply.github.com> Date: Fri, 10 Apr 2026 11:29:15 -0400 Subject: [PATCH] [FR] [DAC] Initial Yaml Support (#5821) * Initial Yaml Support --- CLI.md | 33 ++++++---- detection_rules/action_connector.py | 11 ++++ detection_rules/exception.py | 28 ++++++++- detection_rules/kbwrap.py | 35 ++++++++++- detection_rules/main.py | 97 +++++++++++++++++++++++++---- detection_rules/rule.py | 5 ++ detection_rules/utils.py | 20 +++++- pyproject.toml | 2 +- 8 files changed, 199 insertions(+), 32 deletions(-) diff --git a/CLI.md b/CLI.md index 37960b936..2621da5ef 100644 --- a/CLI.md +++ b/CLI.md @@ -413,21 +413,25 @@ python -m detection_rules kibana import-rules -d test-export-rules -o Toml formatted rule files can also be imported into Kibana through Kibana security app via a consolidated ndjson file which is exported from detection rules. +For this command, **`-d` / `--directory`** selects **input**: directories to load rules from (same as other multi-collection commands). **`--outfile` / `-o`** is the **NDJSON output path** when you are not using YAML mode. **`--save-yaml-dir` / `-syd`** writes **per-rule (and related) YAML files** into that directory instead of producing a single NDJSON file; when `-syd` is set, `-o` is unused. + +Default NDJSON path when `-o` is omitted: `exports/.ndjson` under the detection-rules repository root. + ```console Usage: detection_rules export-rules-from-repo [OPTIONS] Export rule(s) and exception(s) into an importable ndjson file. Options: - -f, --rule-file FILE - -d, --directory DIRECTORY Recursively load rules from a directory - -id, --rule-id TEXT + -f, --rule-file FILE Rule file(s) to load (repeatable) + -d, --directory DIRECTORY Recursively load rules from a directory (repeatable) + -id, --rule-id TEXT Load prebuilt rules matching these IDs (repeatable) -nt, --no-tactic-filename Allow rule filenames without tactic prefix. Use this if rules have been exported with this flag. - -o, --outfile PATH Name of file for exported rules - -r, --replace-id Replace rule IDs with new IDs before export - --stack-version [7.8|7.9|7.10|7.11|7.12|7.13|7.14|7.15|7.16|8.0|8.1|8.2|8.3|8.4|8.5|8.6|8.7|8.8|8.9|8.10|8.11|8.12|8.13|8.14|8.15|8.16|8.17|8.18|9.0] - Downgrade a rule version to be compatible with older instances of Kibana - -s, --skip-unsupported If `--stack-version` is passed, skip rule types which are unsupported (an error will be raised otherwise) + -o, --outfile PATH NDJSON file path for exported rules (ignored if --save-yaml-dir is set) + -syd, --save-yaml-dir PATH Export individual YAML files into this directory instead of NDJSON + -r, --replace-id Replace rule IDs with new UUIDs before export + --stack-version [7.8|...|9.0] Downgrade rule payloads for older Kibana (see `export-rules-from-repo --help` for full list) + -s, --skip-unsupported With `--stack-version`, skip unsupported rule types instead of erroring --include-metadata Add metadata to the exported rules -ac, --include-action-connectors Include Action Connectors in export @@ -500,18 +504,19 @@ Usage: detection_rules kibana export-rules [OPTIONS] Export rules from Kibana. Options: - -d, --directory PATH Directory to export rules to [required] + -d, --directory PATH Directory to write exported rules to [required] -acd, --action-connectors-directory PATH - Directory to export action connectors to + Directory to export action connectors to (defaults from rules config if omitted) -ed, --exceptions-directory PATH - Directory to export exceptions to + Directory to export exceptions to (defaults from rules config if omitted) -da, --default-author TEXT Default author for rules missing one - -r, --rule-id TEXT Optional Rule IDs to restrict export to - -rn, --rule-name TEXT Optional Rule name to restrict export to (KQL, case-insensitive, supports wildcards) + -r, --rule-id TEXT Optional rule ID(s) to restrict export to (repeatable) + -rn, --rule-name TEXT Optional rule name filter (KQL, case-insensitive, wildcards); mutually exclusive with `--rule-id` -ac, --export-action-connectors Include action connectors in export -e, --export-exceptions Include exceptions in export -s, --skip-errors Skip errors when exporting rules + -sy, --save-as-yaml Write rules (and exported exceptions/connectors when requested) as YAML under `--directory` instead of TOML -sv, --strip-version Strip the version fields from all rules -nt, --no-tactic-filename Exclude tactic prefix in exported filenames for rules. Use same flag for import-rules to prevent warnings and disable its unit test. -lc, --local-creation-date Preserve the local creation date of the rule @@ -523,6 +528,8 @@ Options: ``` +**Note:** `kibana export-rules` **`--directory` / `-d`** is the **output** directory only. It is unrelated to **`export-rules-from-repo`**, where **`-d`** means **input** rule directories. + Example of a rule exporting, with errors skipped ``` diff --git a/detection_rules/action_connector.py b/detection_rules/action_connector.py index 7f98f58e2..35aaffe93 100644 --- a/detection_rules/action_connector.py +++ b/detection_rules/action_connector.py @@ -16,6 +16,7 @@ from marshmallow import EXCLUDE from .config import parse_rules_config from .mixins import MarshmallowDataclassMixin from .schemas import definitions +from .utils import ensure_yaml_suffix, save_yaml RULES_CONFIG = parse_rules_config() @@ -111,6 +112,16 @@ class TOMLActionConnector: sorted_dict = dict(sorted(contents_dict.items(), key=lambda item: item[0] != "metadata")) pytoml.dump(sorted_dict, f) # type: ignore[reportUnknownMemberType] + def save_yaml(self, path: Path | None = None) -> None: + """Save the action to a YAML file.""" + target_path = path or self.path + if not target_path: + raise ValueError(f"Can't save action for {self.name} without a path") + api_format = self.contents.to_api_format() + # If single item, write as dict; if multiple, write as list + content = api_format[0] if len(api_format) == 1 else api_format + save_yaml(ensure_yaml_suffix(target_path), content) + def parse_action_connector_results_from_api( results: list[dict[str, Any]], diff --git a/detection_rules/exception.py b/detection_rules/exception.py index f67531a57..37f7f2108 100644 --- a/detection_rules/exception.py +++ b/detection_rules/exception.py @@ -6,7 +6,7 @@ from collections import defaultdict from dataclasses import dataclass -from datetime import datetime +from datetime import UTC, datetime from pathlib import Path from typing import Any, get_args @@ -16,6 +16,7 @@ from marshmallow import EXCLUDE, ValidationError, validates_schema from .config import parse_rules_config from .mixins import MarshmallowDataclassMixin from .schemas import definitions +from .utils import ensure_yaml_suffix, save_yaml RULES_CONFIG = parse_rules_config() @@ -176,8 +177,19 @@ class TOMLExceptionContents(MarshmallowDataclassMixin): # Format date to match schema container = exceptions_dict["container"] - creation_date = datetime.strptime(container["created_at"], "%Y-%m-%dT%H:%M:%S.%fZ").strftime("%Y/%m/%d") # noqa: DTZ007 - updated_date = datetime.strptime(container["updated_at"], "%Y-%m-%dT%H:%M:%S.%fZ").strftime("%Y/%m/%d") # noqa: DTZ007 + now_date = datetime.now(UTC).strftime("%Y/%m/%d") + created_at = container.get("created_at") + updated_at = container.get("updated_at") + creation_date = ( + datetime.strptime(created_at, "%Y-%m-%dT%H:%M:%S.%fZ").strftime("%Y/%m/%d") # noqa: DTZ007 + if created_at + else now_date + ) + updated_date = ( + datetime.strptime(updated_at, "%Y-%m-%dT%H:%M:%S.%fZ").strftime("%Y/%m/%d") # noqa: DTZ007 + if updated_at + else now_date + ) metadata = { "creation_date": creation_date, "list_name": exceptions_dict["container"]["name"], @@ -227,6 +239,16 @@ class TOMLException: sorted_dict = dict(sorted(contents_dict.items(), key=lambda item: item[0] != "metadata")) pytoml.dump(sorted_dict, f) # type: ignore[reportUnknownMemberType] + def save_yaml(self, path: Path | None = None) -> None: + """Save the exception to a YAML file.""" + target_path = path or self.path + if not target_path: + raise ValueError(f"Can't save exception {self.name} without a path") + api_format = self.contents.to_api_format() + # If single item, write as dict; if multiple, write as list + content = api_format[0] if len(api_format) == 1 else api_format + save_yaml(ensure_yaml_suffix(target_path), content) + def parse_exceptions_results_from_api( results: list[dict[str, Any]], diff --git a/detection_rules/kbwrap.py b/detection_rules/kbwrap.py index 517a90a67..3c1d58164 100644 --- a/detection_rules/kbwrap.py +++ b/detection_rules/kbwrap.py @@ -224,6 +224,13 @@ def kibana_import_rules( # noqa: PLR0915 @click.option("--export-action-connectors", "-ac", is_flag=True, help="Include action connectors in export") @click.option("--export-exceptions", "-e", is_flag=True, help="Include exceptions in export") @click.option("--skip-errors", "-s", is_flag=True, help="Skip errors when exporting rules") +@click.option( + "--save-as-yaml", + "-sy", + is_flag=True, + default=False, + help="Save exported rules and objects as YAML into --directory (instead of TOML)", +) @click.option("--strip-version", "-sv", is_flag=True, help="Strip the version fields from all rules") @click.option( "--no-tactic-filename", @@ -263,6 +270,7 @@ def kibana_export_rules( # noqa: PLR0912, PLR0913, PLR0915 export_action_connectors: bool = False, export_exceptions: bool = False, skip_errors: bool = False, + save_as_yaml: bool = False, strip_version: bool = False, no_tactic_filename: bool = False, local_creation_date: bool = False, @@ -272,6 +280,10 @@ def kibana_export_rules( # noqa: PLR0912, PLR0913, PLR0915 load_rule_loading: bool = False, ) -> list[TOMLRule]: """Export rules from Kibana.""" + + def _raise_missing_path(message: str) -> None: + raise ValueError(message) + kibana = ctx.obj["kibana"] kibana_include_details = export_exceptions or export_action_connectors or custom_rules_only or export_query @@ -455,7 +467,14 @@ def kibana_export_rules( # noqa: PLR0912, PLR0913, PLR0915 saved: list[TOMLRule] = [] for rule in exported: try: - rule.save_toml() + if save_as_yaml: + rule_path = rule.path + if isinstance(rule_path, Path): + rule.save_yaml(directory / rule_path.name) + else: + _raise_missing_path(f"Can't save rule {rule.name} ({rule.id}) without a path") + else: + rule.save_toml() except Exception as e: if skip_errors: print(f"- skipping {rule.contents.data.name} - {type(e).__name__}") @@ -468,7 +487,14 @@ def kibana_export_rules( # noqa: PLR0912, PLR0913, PLR0915 saved_exceptions: list[TOMLException] = [] for exception in exceptions: try: - exception.save_toml() + if save_as_yaml: + exception_path = exception.path + if isinstance(exception_path, Path): + exception.save_yaml(directory / exception_path.name) + else: + _raise_missing_path(f"Can't save exception {exception.name} without a path") + else: + exception.save_toml() except Exception as e: if skip_errors: print(f"- skipping {exception.rule_name} - {type(e).__name__}") # type: ignore[reportUnknownMemberType] @@ -481,7 +507,10 @@ def kibana_export_rules( # noqa: PLR0912, PLR0913, PLR0915 saved_action_connectors: list[TOMLActionConnector] = [] for action in action_connectors: try: - action.save_toml() + if save_as_yaml: + action.save_yaml(directory / action.path.name) + else: + action.save_toml() except Exception as e: if skip_errors: print(f"- skipping {action.name} - {type(e).__name__}") diff --git a/detection_rules/main.py b/detection_rules/main.py index 95e541390..baf1d1de8 100644 --- a/detection_rules/main.py +++ b/detection_rules/main.py @@ -21,6 +21,7 @@ from marshmallow_dataclass import class_schema from semver import Version from .action_connector import ( + TOMLActionConnector, TOMLActionConnectorContents, build_action_connector_objects, parse_action_connector_results_from_api, @@ -28,7 +29,7 @@ from .action_connector import ( from .attack import build_threat_map_entry from .cli_utils import multi_collection, rule_prompt from .config import load_current_package_version, parse_rules_config -from .exception import TOMLExceptionContents, build_exception_objects, parse_exceptions_results_from_api +from .exception import TOMLException, TOMLExceptionContents, build_exception_objects, parse_exceptions_results_from_api from .generic_loader import GenericCollection from .misc import ( add_client, @@ -512,6 +513,60 @@ def view_rule( return rule +def _export_rules_as_yaml( # noqa: PLR0913 + rules: RuleCollection, + yaml_directory: Path, + downgrade_version: definitions.SemVer | None = None, + verbose: bool = True, + skip_unsupported: bool = False, + include_metadata: bool = False, + include_action_connectors: bool = False, + include_exceptions: bool = False, +) -> None: + """Export rules and exceptions into a directory of YAML files.""" + from .rule import downgrade_contents_from_rule + + unsupported: list[str] = [] + + for rule in rules: + contents_override = None + if downgrade_version: + try: + contents_override = downgrade_contents_from_rule( + rule, downgrade_version, include_metadata=include_metadata + ) + except ValueError as e: + if skip_unsupported: + unsupported.append(f"{e}: {rule.id} - {rule.name}") + continue + raise + + rule_path = yaml_directory / rulename_to_filename(rule.name) + rule_path.parent.mkdir(parents=True, exist_ok=True) + rule.save_yaml(rule_path, contents_override=contents_override) + + export_types: list[type[Any]] = [] + if include_exceptions: + export_types.append(TOMLException) + if include_action_connectors: + export_types.append(TOMLActionConnector) + + if export_types: + cl = GenericCollection.default() + for d in cl.items: + if any(isinstance(d, export_type) for export_type in export_types): + save_yaml = getattr(d, "save_yaml", None) + if callable(save_yaml) and isinstance(d.path, Path): + _ = save_yaml(yaml_directory / d.path.name) + + if verbose: + click.echo(f"Exported {len(rules) - len(unsupported)} rules into {yaml_directory}") + + if skip_unsupported and unsupported: + unsupported_str = "\n- ".join(unsupported) + click.echo(f"Skipped {len(unsupported)} unsupported rules: \n- {unsupported_str}") + + def _export_rules( # noqa: PLR0913 rules: RuleCollection, outfile: Path, @@ -611,6 +666,13 @@ def _export_rules( # noqa: PLR0913 @click.option( "--include-exceptions", "-e", type=bool, is_flag=True, default=False, help="Include Exceptions Lists in export" ) +@click.option( + "--save-yaml-dir", + "-syd", + type=Path, + required=False, + help="Optional directory to export individual YAML files instead of NDJSON", +) def export_rules_from_repo( # noqa: PLR0913 rules: RuleCollection, outfile: Path, @@ -620,6 +682,7 @@ def export_rules_from_repo( # noqa: PLR0913 include_metadata: bool, include_action_connectors: bool, include_exceptions: bool, + save_yaml_dir: Path | None, ) -> RuleCollection: """Export rule(s) and exception(s) into an importable ndjson file.""" if len(rules) == 0: @@ -636,16 +699,28 @@ def export_rules_from_repo( # noqa: PLR0913 new_contents = dataclasses.replace(rule.contents, data=new_data) rules.add_rule(TOMLRule(contents=new_contents)) - outfile.parent.mkdir(exist_ok=True) - _export_rules( - rules=rules, - outfile=outfile, - downgrade_version=stack_version, - skip_unsupported=skip_unsupported, - include_metadata=include_metadata, - include_action_connectors=include_action_connectors, - include_exceptions=include_exceptions, - ) + if save_yaml_dir: + save_yaml_dir.mkdir(parents=True, exist_ok=True) + _export_rules_as_yaml( + rules=rules, + yaml_directory=save_yaml_dir, + downgrade_version=stack_version, + skip_unsupported=skip_unsupported, + include_metadata=include_metadata, + include_action_connectors=include_action_connectors, + include_exceptions=include_exceptions, + ) + else: + outfile.parent.mkdir(exist_ok=True) + _export_rules( + rules=rules, + outfile=outfile, + downgrade_version=stack_version, + skip_unsupported=skip_unsupported, + include_metadata=include_metadata, + include_action_connectors=include_action_connectors, + include_exceptions=include_exceptions, + ) return rules diff --git a/detection_rules/rule.py b/detection_rules/rule.py index 5afff9b01..b341dff9f 100644 --- a/detection_rules/rule.py +++ b/detection_rules/rule.py @@ -1742,6 +1742,11 @@ class TOMLRule: json.dump(self.contents.to_api_format(include_version=include_version), f, sort_keys=True, indent=2) _ = f.write("\n") + def save_yaml(self, path: Path, contents_override: dict[str, Any] | None = None) -> None: + """Save the rule in YAML format.""" + data = contents_override if contents_override is not None else self.contents.to_api_format() + utils.save_yaml(path.with_suffix(".yaml"), data, use_absolute_path=True) + @dataclass(frozen=True) class DeprecatedRuleContents(BaseRuleContents): diff --git a/detection_rules/utils.py b/detection_rules/utils.py index 80c863b8a..9b5d84db1 100644 --- a/detection_rules/utils.py +++ b/detection_rules/utils.py @@ -27,6 +27,7 @@ from typing import Any import click import eql.utils # type: ignore[reportMissingTypeStubs] import pytoml # type: ignore[reportMissingTypeStubs] +import yaml from eql.utils import load_dump # type: ignore[reportMissingTypeStubs] from github.Repository import Repository @@ -136,6 +137,23 @@ def save_etc_dump(contents: dict[str, Any], path: list[str], sort_keys: bool = T eql.utils.save_dump(contents, path) # type: ignore[reportUnknownVariableType] +def ensure_yaml_suffix(path: Path) -> Path: + """If ``path`` has no YAML extension, use ``.yaml``; keep ``.yaml`` / ``.yml`` unchanged.""" + if path.suffix in (".yaml", ".yml"): + return path + return path.with_suffix(".yaml") + + +def save_yaml(path: Path, data: Any, *, use_absolute_path: bool = False) -> None: + """Write ``data`` as YAML with sorted keys, block style, UTF-8, and a trailing newline.""" + out_path = path.absolute() if use_absolute_path else path + with out_path.open("w", encoding="utf-8", newline="\n") as f: + output = yaml.safe_dump(data, sort_keys=True, default_flow_style=False) + _ = f.write(output) + if not output.endswith("\n"): + _ = f.write("\n") + + # Top-level _config.yaml key -> DR_BYPASS_* env var set when true at load time OPTIONAL_ELASTIC_VALIDATION_BYPASS_ENV: dict[str, str] = { "bypass_note_validation_and_parse": "DR_BYPASS_NOTE_VALIDATION_AND_PARSE", @@ -362,7 +380,7 @@ def load_rule_contents(rule_file: Path, single_only: bool = False) -> list[Any]: return contents or [{}] if extension == ".toml": rule = pytoml.loads(raw_text) # type: ignore[reportUnknownVariableType] - elif extension.lower() in ("yaml", "yml"): + elif extension.lower() in (".yaml", ".yml"): rule = load_dump(str(rule_file)) else: return [] diff --git a/pyproject.toml b/pyproject.toml index 3338d0704..19db32ee0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "detection_rules" -version = "1.6.16" +version = "1.6.17" description = "Detection Rules is the home for rules used by Elastic Security. This repository is used for the development, maintenance, testing, validation, and release of rules for Elastic Security’s Detection Engine." readme = "README.md" requires-python = ">=3.12"