From 40015070b412bc1b58afa276b93ec30ef5ee4b2f Mon Sep 17 00:00:00 2001 From: Mika Ayenson Date: Fri, 3 May 2024 13:43:22 -0500 Subject: [PATCH] [FR] Add ability to generate hunt index (#3643) (cherry picked from commit c8c8c96956a8db5950c58dd8b9fe32c2ef1f9ec5) --- README.md | 6 +++--- hunting/generate_markdown.py | 24 +++++++++++++++++++++++- hunting/index.md | 6 +++--- 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 281213d3d..76418901e 100644 --- a/README.md +++ b/README.md @@ -28,9 +28,9 @@ Detection Rules contains more than just static rule files. This repository also |------------------------------------------------ |------------------------------------------------------------------------------------ | | [`detection_rules/`](detection_rules) | Python module for rule parsing, validating and packaging | | [`etc/`](detection_rules/etc) | Miscellaneous files, such as ECS and Beats schemas | -|[`hunting`](./hunting/)|Root directory where threat hunting queries are stored| -| [`kibana/`](lib/kibana) | Python library for handling the API calls to Kibana and the Detection Engine | -| [`kql/`](lib/kql) | Python library for parsing and validating Kibana Query Language | +| [`hunting`](./hunting/) | Root directory where threat hunting queries are stored | +| [`kibana/`](lib/kibana) | Python library for handling the API calls to Kibana and the Detection Engine | +| [`kql/`](lib/kql) | Python library for parsing and validating Kibana Query Language | | [`rta/`](rta) | Red Team Automation code used to emulate attacker techniques, used for rule testing | | [`rules/`](rules) | Root directory where rules are stored | | [`rules_building_block/`](rules_building_block) | Root directory where building block rules are stored | diff --git a/hunting/generate_markdown.py b/hunting/generate_markdown.py index a1da0afc8..e0a9c3947 100644 --- a/hunting/generate_markdown.py +++ b/hunting/generate_markdown.py @@ -11,6 +11,8 @@ from pathlib import Path from typing import List, Optional HUNTING_DIR = Path(__file__).parent +ATLAS_URL = "https://atlas.mitre.org/techniques/" +ATTACK_URL = "https://attack.mitre.org/techniques/" @dataclass @@ -57,7 +59,9 @@ def convert_toml_to_markdown(hunt_config: Hunt, file_path: Path) -> str: markdown += "## Notes\n\n" + "\n".join(f"- {note}" for note in hunt_config.notes) if hunt_config.mitre: markdown += "\n## MITRE ATT&CK Techniques\n\n" + "\n".join( - f"- [{tech}](https://atlas.mitre.org/techniques/{tech})\n" for tech in hunt_config.mitre + f"- [{tech}]({ATLAS_URL if tech.startswith('AML') else ATTACK_URL}" + f"{tech.replace('.', '/') if tech.startswith('T') else tech})\n" + for tech in hunt_config.mitre ) if hunt_config.references: markdown += "\n## References\n\n" + "\n".join(f"- {ref}" for ref in hunt_config.references) @@ -68,12 +72,30 @@ def convert_toml_to_markdown(hunt_config: Hunt, file_path: Path) -> str: def process_toml_files(base_path: Path) -> None: """Process all TOML files in the directory recursively and convert them to Markdown.""" hunts = load_all_toml(base_path) + index_content = "# List of Available Queries\n\nHere are the queries currently available:\n\n" + directories = {} + for hunt_config, toml_file in hunts: markdown_content = convert_toml_to_markdown(hunt_config, toml_file) markdown_path = toml_file.parent.parent / "docs" / f"{toml_file.stem}.md" markdown_path.parent.mkdir(parents=True, exist_ok=True) markdown_path.write_text(markdown_content, encoding="utf-8") print(f"Markdown generated: {markdown_path}") + relative_path = markdown_path.relative_to(base_path) + folder_name = toml_file.parent.parent.name + directories.setdefault(folder_name, []).append((relative_path, hunt_config.name, hunt_config.language)) + + # Build index content + for folder, files in sorted(directories.items()): + index_content += f"## {folder}\n" + for file_path, rule_name, language in sorted(files): + index_path = "./" + str(file_path) + index_content += f"- [{rule_name}]({index_path}) ({language})\n" + + # Write the index file at the base directory level + index_path = base_path / "index.md" + index_path.write_text(index_content, encoding="utf-8") + print(f"Index Markdown generated at: {index_path}") if __name__ == "__main__": diff --git a/hunting/index.md b/hunting/index.md index 5fb35f16e..564a9d435 100644 --- a/hunting/index.md +++ b/hunting/index.md @@ -3,6 +3,6 @@ Here are the queries currently available: ## llm -- [Denial of Service or Resource Exhaustion Attacks Detection](llm/queries/docs/llm_dos_resource_exhaustion_detection.md) (ES|QL) -- [Monitoring for Latency Anomalies](llm/queries/docs/llm_latency_anomalies_detection.md) (ES|QL) -- [Sensitive Content Refusal Detection](llm/queries/docs/llm_sensitive_content_refusal_detection.md) (ES|QL) +- [Denial of Service or Resource Exhaustion Attacks Detection](./llm/docs/llm_dos_resource_exhaustion_detection.md) (ES|QL) +- [Monitoring for Latency Anomalies](./llm/docs/llm_latency_anomalies_detection.md) (ES|QL) +- [Sensitive Content Refusal Detection](./llm/docs/llm_sensitive_content_refusal_detection.md) (ES|QL)