[FR] [DAC] Initial Yaml Support (#5821)

* Initial Yaml Support

CLI.md

@@ -413,21 +413,25 @@ python -m detection_rules kibana import-rules -d test-export-rules -o
 Toml formatted rule files can also be imported into Kibana through Kibana security app via a consolidated ndjson file
 which is exported from detection rules.
 
+For this command, **`-d` / `--directory`** selects **input**: directories to load rules from (same as other multi-collection commands). **`--outfile` / `-o`** is the **NDJSON output path** when you are not using YAML mode. **`--save-yaml-dir` / `-syd`** writes **per-rule (and related) YAML files** into that directory instead of producing a single NDJSON file; when `-syd` is set, `-o` is unused.
+
+Default NDJSON path when `-o` is omitted: `exports/<timestamp>.ndjson` under the detection-rules repository root.
+
 ```console
 Usage: detection_rules export-rules-from-repo [OPTIONS]
 
   Export rule(s) and exception(s) into an importable ndjson file.
 
 Options:
-  -f, --rule-file FILE
-  -d, --directory DIRECTORY       Recursively load rules from a directory
-  -id, --rule-id TEXT
+  -f, --rule-file FILE            Rule file(s) to load (repeatable)
+  -d, --directory DIRECTORY      Recursively load rules from a directory (repeatable)
+  -id, --rule-id TEXT             Load prebuilt rules matching these IDs (repeatable)
   -nt, --no-tactic-filename       Allow rule filenames without tactic prefix. Use this if rules have been exported with this flag.
-  -o, --outfile PATH              Name of file for exported rules
-  -r, --replace-id                Replace rule IDs with new IDs before export
-  --stack-version [7.8|7.9|7.10|7.11|7.12|7.13|7.14|7.15|7.16|8.0|8.1|8.2|8.3|8.4|8.5|8.6|8.7|8.8|8.9|8.10|8.11|8.12|8.13|8.14|8.15|8.16|8.17|8.18|9.0]
-                                  Downgrade a rule version to be compatible with older instances of Kibana
-  -s, --skip-unsupported          If `--stack-version` is passed, skip rule types which are unsupported (an error will be raised otherwise)
+  -o, --outfile PATH              NDJSON file path for exported rules (ignored if --save-yaml-dir is set)
+  -syd, --save-yaml-dir PATH      Export individual YAML files into this directory instead of NDJSON
+  -r, --replace-id                Replace rule IDs with new UUIDs before export
+  --stack-version [7.8|...|9.0]   Downgrade rule payloads for older Kibana (see `export-rules-from-repo --help` for full list)
+  -s, --skip-unsupported          With `--stack-version`, skip unsupported rule types instead of erroring
   --include-metadata              Add metadata to the exported rules
   -ac, --include-action-connectors
                                   Include Action Connectors in export
@@ -500,18 +504,19 @@ Usage: detection_rules kibana export-rules [OPTIONS]
   Export rules from Kibana.
 
 Options:
-  -d, --directory PATH            Directory to export rules to  [required]
+  -d, --directory PATH            Directory to write exported rules to  [required]
   -acd, --action-connectors-directory PATH
-                                  Directory to export action connectors to
+                                  Directory to export action connectors to (defaults from rules config if omitted)
   -ed, --exceptions-directory PATH
-                                  Directory to export exceptions to
+                                  Directory to export exceptions to (defaults from rules config if omitted)
   -da, --default-author TEXT      Default author for rules missing one
-  -r, --rule-id TEXT              Optional Rule IDs to restrict export to
-  -rn, --rule-name TEXT           Optional Rule name to restrict export to (KQL, case-insensitive, supports wildcards)
+  -r, --rule-id TEXT              Optional rule ID(s) to restrict export to (repeatable)
+  -rn, --rule-name TEXT           Optional rule name filter (KQL, case-insensitive, wildcards); mutually exclusive with `--rule-id`
   -ac, --export-action-connectors
                                   Include action connectors in export
   -e, --export-exceptions         Include exceptions in export
   -s, --skip-errors               Skip errors when exporting rules
+  -sy, --save-as-yaml             Write rules (and exported exceptions/connectors when requested) as YAML under `--directory` instead of TOML
   -sv, --strip-version            Strip the version fields from all rules
   -nt, --no-tactic-filename       Exclude tactic prefix in exported filenames for rules. Use same flag for import-rules to prevent warnings and disable its unit test.
   -lc, --local-creation-date      Preserve the local creation date of the rule
@@ -523,6 +528,8 @@ Options:
 
 ```
 
+**Note:** `kibana export-rules` **`--directory` / `-d`** is the **output** directory only. It is unrelated to **`export-rules-from-repo`**, where **`-d`** means **input** rule directories.
+
 Example of a rule exporting, with errors skipped
 
 ```

detection_rules/action_connector.py

@@ -16,6 +16,7 @@ from marshmallow import EXCLUDE
 from .config import parse_rules_config
 from .mixins import MarshmallowDataclassMixin
 from .schemas import definitions
+from .utils import ensure_yaml_suffix, save_yaml
 
 RULES_CONFIG = parse_rules_config()
 
@@ -111,6 +112,16 @@ class TOMLActionConnector:
             sorted_dict = dict(sorted(contents_dict.items(), key=lambda item: item[0] != "metadata"))
             pytoml.dump(sorted_dict, f)  # type: ignore[reportUnknownMemberType]
 
+    def save_yaml(self, path: Path | None = None) -> None:
+        """Save the action to a YAML file."""
+        target_path = path or self.path
+        if not target_path:
+            raise ValueError(f"Can't save action for {self.name} without a path")
+        api_format = self.contents.to_api_format()
+        # If single item, write as dict; if multiple, write as list
+        content = api_format[0] if len(api_format) == 1 else api_format
+        save_yaml(ensure_yaml_suffix(target_path), content)
+
 
 def parse_action_connector_results_from_api(
     results: list[dict[str, Any]],

detection_rules/exception.py

@@ -6,7 +6,7 @@
 
 from collections import defaultdict
 from dataclasses import dataclass
-from datetime import datetime
+from datetime import UTC, datetime
 from pathlib import Path
 from typing import Any, get_args
 
@@ -16,6 +16,7 @@ from marshmallow import EXCLUDE, ValidationError, validates_schema
 from .config import parse_rules_config
 from .mixins import MarshmallowDataclassMixin
 from .schemas import definitions
+from .utils import ensure_yaml_suffix, save_yaml
 
 RULES_CONFIG = parse_rules_config()
 
@@ -176,8 +177,19 @@ class TOMLExceptionContents(MarshmallowDataclassMixin):
 
         # Format date to match schema
         container = exceptions_dict["container"]
-        creation_date = datetime.strptime(container["created_at"], "%Y-%m-%dT%H:%M:%S.%fZ").strftime("%Y/%m/%d")  # noqa: DTZ007
-        updated_date = datetime.strptime(container["updated_at"], "%Y-%m-%dT%H:%M:%S.%fZ").strftime("%Y/%m/%d")  # noqa: DTZ007
+        now_date = datetime.now(UTC).strftime("%Y/%m/%d")
+        created_at = container.get("created_at")
+        updated_at = container.get("updated_at")
+        creation_date = (
+            datetime.strptime(created_at, "%Y-%m-%dT%H:%M:%S.%fZ").strftime("%Y/%m/%d")  # noqa: DTZ007
+            if created_at
+            else now_date
+        )
+        updated_date = (
+            datetime.strptime(updated_at, "%Y-%m-%dT%H:%M:%S.%fZ").strftime("%Y/%m/%d")  # noqa: DTZ007
+            if updated_at
+            else now_date
+        )
         metadata = {
             "creation_date": creation_date,
             "list_name": exceptions_dict["container"]["name"],
@@ -227,6 +239,16 @@ class TOMLException:
             sorted_dict = dict(sorted(contents_dict.items(), key=lambda item: item[0] != "metadata"))
             pytoml.dump(sorted_dict, f)  # type: ignore[reportUnknownMemberType]
 
+    def save_yaml(self, path: Path | None = None) -> None:
+        """Save the exception to a YAML file."""
+        target_path = path or self.path
+        if not target_path:
+            raise ValueError(f"Can't save exception {self.name} without a path")
+        api_format = self.contents.to_api_format()
+        # If single item, write as dict; if multiple, write as list
+        content = api_format[0] if len(api_format) == 1 else api_format
+        save_yaml(ensure_yaml_suffix(target_path), content)
+
 
 def parse_exceptions_results_from_api(
     results: list[dict[str, Any]],

detection_rules/kbwrap.py

@@ -224,6 +224,13 @@ def kibana_import_rules(  # noqa: PLR0915
 @click.option("--export-action-connectors", "-ac", is_flag=True, help="Include action connectors in export")
 @click.option("--export-exceptions", "-e", is_flag=True, help="Include exceptions in export")
 @click.option("--skip-errors", "-s", is_flag=True, help="Skip errors when exporting rules")
+@click.option(
+    "--save-as-yaml",
+    "-sy",
+    is_flag=True,
+    default=False,
+    help="Save exported rules and objects as YAML into --directory (instead of TOML)",
+)
 @click.option("--strip-version", "-sv", is_flag=True, help="Strip the version fields from all rules")
 @click.option(
     "--no-tactic-filename",
@@ -263,6 +270,7 @@ def kibana_export_rules(  # noqa: PLR0912, PLR0913, PLR0915
     export_action_connectors: bool = False,
     export_exceptions: bool = False,
     skip_errors: bool = False,
+    save_as_yaml: bool = False,
     strip_version: bool = False,
     no_tactic_filename: bool = False,
     local_creation_date: bool = False,
@@ -272,6 +280,10 @@ def kibana_export_rules(  # noqa: PLR0912, PLR0913, PLR0915
     load_rule_loading: bool = False,
 ) -> list[TOMLRule]:
     """Export rules from Kibana."""
+
+    def _raise_missing_path(message: str) -> None:
+        raise ValueError(message)
+
     kibana = ctx.obj["kibana"]
     kibana_include_details = export_exceptions or export_action_connectors or custom_rules_only or export_query
 
@@ -455,7 +467,14 @@ def kibana_export_rules(  # noqa: PLR0912, PLR0913, PLR0915
     saved: list[TOMLRule] = []
     for rule in exported:
         try:
-            rule.save_toml()
+            if save_as_yaml:
+                rule_path = rule.path
+                if isinstance(rule_path, Path):
+                    rule.save_yaml(directory / rule_path.name)
+                else:
+                    _raise_missing_path(f"Can't save rule {rule.name} ({rule.id}) without a path")
+            else:
+                rule.save_toml()
         except Exception as e:
             if skip_errors:
                 print(f"- skipping {rule.contents.data.name} - {type(e).__name__}")
@@ -468,7 +487,14 @@ def kibana_export_rules(  # noqa: PLR0912, PLR0913, PLR0915
     saved_exceptions: list[TOMLException] = []
     for exception in exceptions:
         try:
-            exception.save_toml()
+            if save_as_yaml:
+                exception_path = exception.path
+                if isinstance(exception_path, Path):
+                    exception.save_yaml(directory / exception_path.name)
+                else:
+                    _raise_missing_path(f"Can't save exception {exception.name} without a path")
+            else:
+                exception.save_toml()
         except Exception as e:
             if skip_errors:
                 print(f"- skipping {exception.rule_name} - {type(e).__name__}")  # type: ignore[reportUnknownMemberType]
@@ -481,7 +507,10 @@ def kibana_export_rules(  # noqa: PLR0912, PLR0913, PLR0915
     saved_action_connectors: list[TOMLActionConnector] = []
     for action in action_connectors:
         try:
-            action.save_toml()
+            if save_as_yaml:
+                action.save_yaml(directory / action.path.name)
+            else:
+                action.save_toml()
         except Exception as e:
             if skip_errors:
                 print(f"- skipping {action.name} - {type(e).__name__}")

detection_rules/main.py

@@ -21,6 +21,7 @@ from marshmallow_dataclass import class_schema
 from semver import Version
 
 from .action_connector import (
+    TOMLActionConnector,
     TOMLActionConnectorContents,
     build_action_connector_objects,
     parse_action_connector_results_from_api,
@@ -28,7 +29,7 @@ from .action_connector import (
 from .attack import build_threat_map_entry
 from .cli_utils import multi_collection, rule_prompt
 from .config import load_current_package_version, parse_rules_config
-from .exception import TOMLExceptionContents, build_exception_objects, parse_exceptions_results_from_api
+from .exception import TOMLException, TOMLExceptionContents, build_exception_objects, parse_exceptions_results_from_api
 from .generic_loader import GenericCollection
 from .misc import (
     add_client,
@@ -512,6 +513,60 @@ def view_rule(
     return rule
 
 
+def _export_rules_as_yaml(  # noqa: PLR0913
+    rules: RuleCollection,
+    yaml_directory: Path,
+    downgrade_version: definitions.SemVer | None = None,
+    verbose: bool = True,
+    skip_unsupported: bool = False,
+    include_metadata: bool = False,
+    include_action_connectors: bool = False,
+    include_exceptions: bool = False,
+) -> None:
+    """Export rules and exceptions into a directory of YAML files."""
+    from .rule import downgrade_contents_from_rule
+
+    unsupported: list[str] = []
+
+    for rule in rules:
+        contents_override = None
+        if downgrade_version:
+            try:
+                contents_override = downgrade_contents_from_rule(
+                    rule, downgrade_version, include_metadata=include_metadata
+                )
+            except ValueError as e:
+                if skip_unsupported:
+                    unsupported.append(f"{e}: {rule.id} - {rule.name}")
+                    continue
+                raise
+
+        rule_path = yaml_directory / rulename_to_filename(rule.name)
+        rule_path.parent.mkdir(parents=True, exist_ok=True)
+        rule.save_yaml(rule_path, contents_override=contents_override)
+
+    export_types: list[type[Any]] = []
+    if include_exceptions:
+        export_types.append(TOMLException)
+    if include_action_connectors:
+        export_types.append(TOMLActionConnector)
+
+    if export_types:
+        cl = GenericCollection.default()
+        for d in cl.items:
+            if any(isinstance(d, export_type) for export_type in export_types):
+                save_yaml = getattr(d, "save_yaml", None)
+                if callable(save_yaml) and isinstance(d.path, Path):
+                    _ = save_yaml(yaml_directory / d.path.name)
+
+    if verbose:
+        click.echo(f"Exported {len(rules) - len(unsupported)} rules into {yaml_directory}")
+
+        if skip_unsupported and unsupported:
+            unsupported_str = "\n- ".join(unsupported)
+            click.echo(f"Skipped {len(unsupported)} unsupported rules: \n- {unsupported_str}")
+
+
 def _export_rules(  # noqa: PLR0913
     rules: RuleCollection,
     outfile: Path,
@@ -611,6 +666,13 @@ def _export_rules(  # noqa: PLR0913
 @click.option(
     "--include-exceptions", "-e", type=bool, is_flag=True, default=False, help="Include Exceptions Lists in export"
 )
+@click.option(
+    "--save-yaml-dir",
+    "-syd",
+    type=Path,
+    required=False,
+    help="Optional directory to export individual YAML files instead of NDJSON",
+)
 def export_rules_from_repo(  # noqa: PLR0913
     rules: RuleCollection,
     outfile: Path,
@@ -620,6 +682,7 @@ def export_rules_from_repo(  # noqa: PLR0913
     include_metadata: bool,
     include_action_connectors: bool,
     include_exceptions: bool,
+    save_yaml_dir: Path | None,
 ) -> RuleCollection:
     """Export rule(s) and exception(s) into an importable ndjson file."""
     if len(rules) == 0:
@@ -636,16 +699,28 @@ def export_rules_from_repo(  # noqa: PLR0913
             new_contents = dataclasses.replace(rule.contents, data=new_data)
             rules.add_rule(TOMLRule(contents=new_contents))
 
-    outfile.parent.mkdir(exist_ok=True)
-    _export_rules(
-        rules=rules,
-        outfile=outfile,
-        downgrade_version=stack_version,
-        skip_unsupported=skip_unsupported,
-        include_metadata=include_metadata,
-        include_action_connectors=include_action_connectors,
-        include_exceptions=include_exceptions,
-    )
+    if save_yaml_dir:
+        save_yaml_dir.mkdir(parents=True, exist_ok=True)
+        _export_rules_as_yaml(
+            rules=rules,
+            yaml_directory=save_yaml_dir,
+            downgrade_version=stack_version,
+            skip_unsupported=skip_unsupported,
+            include_metadata=include_metadata,
+            include_action_connectors=include_action_connectors,
+            include_exceptions=include_exceptions,
+        )
+    else:
+        outfile.parent.mkdir(exist_ok=True)
+        _export_rules(
+            rules=rules,
+            outfile=outfile,
+            downgrade_version=stack_version,
+            skip_unsupported=skip_unsupported,
+            include_metadata=include_metadata,
+            include_action_connectors=include_action_connectors,
+            include_exceptions=include_exceptions,
+        )
 
     return rules
 

detection_rules/rule.py

@@ -1742,6 +1742,11 @@ class TOMLRule:
             json.dump(self.contents.to_api_format(include_version=include_version), f, sort_keys=True, indent=2)
             _ = f.write("\n")
 
+    def save_yaml(self, path: Path, contents_override: dict[str, Any] | None = None) -> None:
+        """Save the rule in YAML format."""
+        data = contents_override if contents_override is not None else self.contents.to_api_format()
+        utils.save_yaml(path.with_suffix(".yaml"), data, use_absolute_path=True)
+
 
 @dataclass(frozen=True)
 class DeprecatedRuleContents(BaseRuleContents):

detection_rules/utils.py

@@ -27,6 +27,7 @@ from typing import Any
 import click
 import eql.utils  # type: ignore[reportMissingTypeStubs]
 import pytoml  # type: ignore[reportMissingTypeStubs]
+import yaml
 from eql.utils import load_dump  # type: ignore[reportMissingTypeStubs]
 from github.Repository import Repository
 
@@ -136,6 +137,23 @@ def save_etc_dump(contents: dict[str, Any], path: list[str], sort_keys: bool = T
         eql.utils.save_dump(contents, path)  # type: ignore[reportUnknownVariableType]
 
 
+def ensure_yaml_suffix(path: Path) -> Path:
+    """If ``path`` has no YAML extension, use ``.yaml``; keep ``.yaml`` / ``.yml`` unchanged."""
+    if path.suffix in (".yaml", ".yml"):
+        return path
+    return path.with_suffix(".yaml")
+
+
+def save_yaml(path: Path, data: Any, *, use_absolute_path: bool = False) -> None:
+    """Write ``data`` as YAML with sorted keys, block style, UTF-8, and a trailing newline."""
+    out_path = path.absolute() if use_absolute_path else path
+    with out_path.open("w", encoding="utf-8", newline="\n") as f:
+        output = yaml.safe_dump(data, sort_keys=True, default_flow_style=False)
+        _ = f.write(output)
+        if not output.endswith("\n"):
+            _ = f.write("\n")
+
+
 # Top-level _config.yaml key -> DR_BYPASS_* env var set when true at load time
 OPTIONAL_ELASTIC_VALIDATION_BYPASS_ENV: dict[str, str] = {
     "bypass_note_validation_and_parse": "DR_BYPASS_NOTE_VALIDATION_AND_PARSE",
@@ -362,7 +380,7 @@ def load_rule_contents(rule_file: Path, single_only: bool = False) -> list[Any]:
         return contents or [{}]
     if extension == ".toml":
         rule = pytoml.loads(raw_text)  # type: ignore[reportUnknownVariableType]
-    elif extension.lower() in ("yaml", "yml"):
+    elif extension.lower() in (".yaml", ".yml"):
         rule = load_dump(str(rule_file))
     else:
         return []

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "detection_rules"
-version = "1.6.16"
+version = "1.6.17"
 description = "Detection Rules is the home for rules used by Elastic Security. This repository is used for the development, maintenance, testing, validation, and release of rules for Elastic Security’s Detection Engine."
 readme = "README.md"
 requires-python = ">=3.12"