From 50e23ba242efcdf13e1fefd1d3b7757ccdd02d83 Mon Sep 17 00:00:00 2001 From: Terrance DeJesus <99630311+terrancedejesus@users.noreply.github.com> Date: Thu, 3 Oct 2024 12:47:40 -0400 Subject: [PATCH] [Hunting] Re-factor Hunting Library Code (#4085) * updating python code for hunting library * fixed okta queries; added MITRE search capability * fixed hunting unit test imports * fixed duplicate UUID; fixed duplicate index entry bug * fixed technique finding sub-technique in search * added more unit tests * linted * flake errors addressed; fixed unit test import; fixed markdown generate bug * added description for generate-markdown command * updated README * adjusted YAML index, adjusted code for index changes * adjusted relative imports; updated CODEOWNERS * adding updates; moving to different branch for main dependencies * finished run-query command; made some code adjustments * removed some comments * revised makefile; fixed unit tests; adjusted detection rules pyproject * updated README * updated README * adjusted unit tests; adjusted hunt guidelines; updated makefile; adjusted several commands * adjusted package to be more object-oriented * removed unused variable * Add simple breakdown stats * addressed feedback; added keyword option for search * Update hunting/README.md Co-authored-by: Mika Ayenson * Update detection_rules/etc/test_hunting_cli.bash Co-authored-by: Eric Forte <119343520+eric-forte-elastic@users.noreply.github.com> * addressing feedback * addressed feedback * added message for unknown index; fixed function call * fixed search command * fixed flake error --------- Co-authored-by: Mika Ayenson Co-authored-by: Mika Ayenson Co-authored-by: Eric Forte <119343520+eric-forte-elastic@users.noreply.github.com> --- .github/CODEOWNERS | 19 +- .../hunt_new_guidelines.md | 19 +- .../hunt_tuning_guidelines.md | 25 +- .github/paths-labeller.yml | 2 + Makefile | 10 + README.md | 4 +- detection_rules/etc/test_hunting_cli.bash | 34 ++ hunting/README.md | 84 ++- hunting/__main__.py | 257 ++++++++ ...ry_multi_region_get_service_quota_calls.md | 58 -- ...lic_bucket_rapid_object_access_attempts.md | 2 +- ...c_bucket_rapid_object_access_attempts.toml | 2 +- hunting/definitions.py | 36 ++ hunting/generate_markdown.py | 140 ----- hunting/index.md | 107 ++-- hunting/index.yml | 561 ++++++++++++++++++ hunting/markdown.py | 144 +++++ ...ial_access_mfa_bombing_push_notications.md | 0 ...t_password_requests_for_different_users.md | 0 ...s_token_retrieval_via_public_client_app.md | 0 ...cation_sso_authentication_repeat_source.md | 0 ...eported_for_oauth_access_tokens_granted.md | 0 ...uth_access_token_granted_by_application.md | 0 ...gher_than_average_failed_authentication.md | 58 -- ...ss_password_spraying_from_repeat_source.md | 53 -- ...gher_than_average_failed_authentication.md | 59 -- ...tence_rare_tld_with_user_authentication.md | 48 -- ...gher_than_average_failed_authentication.md | 0 ...nitial_access_impossible_travel_sign_on.md | 0 ...ss_password_spraying_from_repeat_source.md | 0 ..._multi_factor_push_notification_bombing.md | 0 ...ce_rare_domain_with_user_authentication.md | 0 ...l_access_mfa_bombing_push_notications.toml | 0 ...password_requests_for_different_users.toml | 0 ...token_retrieval_via_public_client_app.toml | 0 ...tion_sso_authentication_repeat_source.toml | 0 ...orted_for_oauth_access_tokens_granted.toml | 0 ...h_access_token_granted_by_application.toml | 0 ...er_than_average_failed_authentication.toml | 0 ...tial_access_impossible_travel_sign_on.toml | 0 ..._password_spraying_from_repeat_source.toml | 0 ...ulti_factor_push_notification_bombing.toml | 0 ..._rare_domain_with_user_authentication.toml | 0 hunting/run.py | 76 +++ hunting/search.py | 188 ++++++ hunting/utils.py | 134 +++++ pyproject.toml | 3 +- tests/test_hunt_data.py | 64 +- 48 files changed, 1659 insertions(+), 528 deletions(-) create mode 100755 detection_rules/etc/test_hunting_cli.bash create mode 100644 hunting/__main__.py delete mode 100644 hunting/aws/docs/ec2_discovery_multi_region_get_service_quota_calls.md create mode 100644 hunting/definitions.py delete mode 100644 hunting/generate_markdown.py create mode 100644 hunting/index.yml create mode 100644 hunting/markdown.py rename hunting/okta/docs/{docs => }/credential_access_mfa_bombing_push_notications.md (100%) rename hunting/okta/docs/{docs => }/credential_access_rapid_reset_password_requests_for_different_users.md (100%) rename hunting/okta/docs/{docs => }/defense_evasion_failed_oauth_access_token_retrieval_via_public_client_app.md (100%) rename hunting/okta/docs/{docs => }/defense_evasion_multiple_application_sso_authentication_repeat_source.md (100%) rename hunting/okta/docs/{docs => }/defense_evasion_multiple_client_sources_reported_for_oauth_access_tokens_granted.md (100%) rename hunting/okta/docs/{docs => }/defense_evasion_rare_oauth_access_token_granted_by_application.md (100%) delete mode 100644 hunting/okta/docs/docs/credential_access_hgher_than_average_failed_authentication.md delete mode 100644 hunting/okta/docs/docs/credential_access_password_spraying_from_repeat_source.md delete mode 100644 hunting/okta/docs/docs/initial_access_hgher_than_average_failed_authentication.md delete mode 100644 hunting/okta/docs/docs/persistence_rare_tld_with_user_authentication.md rename hunting/okta/docs/{docs => }/initial_access_higher_than_average_failed_authentication.md (100%) rename hunting/okta/docs/{docs => }/initial_access_impossible_travel_sign_on.md (100%) rename hunting/okta/docs/{docs => }/initial_access_password_spraying_from_repeat_source.md (100%) rename hunting/okta/docs/{docs => }/persistence_multi_factor_push_notification_bombing.md (100%) rename hunting/okta/docs/{docs => }/persistence_rare_domain_with_user_authentication.md (100%) rename hunting/okta/{docs => }/queries/credential_access_mfa_bombing_push_notications.toml (100%) rename hunting/okta/{docs => }/queries/credential_access_rapid_reset_password_requests_for_different_users.toml (100%) rename hunting/okta/{docs => }/queries/defense_evasion_failed_oauth_access_token_retrieval_via_public_client_app.toml (100%) rename hunting/okta/{docs => }/queries/defense_evasion_multiple_application_sso_authentication_repeat_source.toml (100%) rename hunting/okta/{docs => }/queries/defense_evasion_multiple_client_sources_reported_for_oauth_access_tokens_granted.toml (100%) rename hunting/okta/{docs => }/queries/defense_evasion_rare_oauth_access_token_granted_by_application.toml (100%) rename hunting/okta/{docs => }/queries/initial_access_higher_than_average_failed_authentication.toml (100%) rename hunting/okta/{docs => }/queries/initial_access_impossible_travel_sign_on.toml (100%) rename hunting/okta/{docs => }/queries/initial_access_password_spraying_from_repeat_source.toml (100%) rename hunting/okta/{docs => }/queries/persistence_multi_factor_push_notification_bombing.toml (100%) rename hunting/okta/{docs => }/queries/persistence_rare_domain_with_user_authentication.toml (100%) create mode 100644 hunting/run.py create mode 100644 hunting/search.py create mode 100644 hunting/utils.py diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index be433830b..6a9941ede 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,14 +1,15 @@ # detection-rules code owners # POC: Elastic Security Intelligence and Analytics Team -tests/**/*.py @mikaayenson @eric-forte-elastic @terrancedejesus -detection_rules/ @mikaayenson @eric-forte-elastic @terrancedejesus -tests/ @mikaayenson @eric-forte-elastic @terrancedejesus -lib/ @mikaayenson @eric-forte-elastic @terrancedejesus -rta/ @mikaayenson @eric-forte-elastic @terrancedejesus +tests/**/*.py @mikaayenson @eric-forte-elastic @terrancedejesus +detection_rules/ @mikaayenson @eric-forte-elastic @terrancedejesus +tests/ @mikaayenson @eric-forte-elastic @terrancedejesus +lib/ @mikaayenson @eric-forte-elastic @terrancedejesus +rta/ @mikaayenson @eric-forte-elastic @terrancedejesus +hunting/ @mikaayenson @eric-forte-elastic @terrancedejesus # skip rta-mapping to avoid the spam -detection_rules/etc/packages.yaml @mikaayenson @eric-forte-elastic @terrancedejesus -detection_rules/etc/*.json @mikaayenson @eric-forte-elastic @terrancedejesus -detection_rules/etc/*.json @mikaayenson @eric-forte-elastic @terrancedejesus -detection_rules/etc/*/* @mikaayenson @eric-forte-elastic @terrancedejesus +detection_rules/etc/packages.yaml @mikaayenson @eric-forte-elastic @terrancedejesus +detection_rules/etc/*.json @mikaayenson @eric-forte-elastic @terrancedejesus +detection_rules/etc/*.json @mikaayenson @eric-forte-elastic @terrancedejesus +detection_rules/etc/*/* @mikaayenson @eric-forte-elastic @terrancedejesus diff --git a/.github/PULL_REQUEST_GUIDELINES/hunt_new_guidelines.md b/.github/PULL_REQUEST_GUIDELINES/hunt_new_guidelines.md index 7956a5b5b..5b0293765 100644 --- a/.github/PULL_REQUEST_GUIDELINES/hunt_new_guidelines.md +++ b/.github/PULL_REQUEST_GUIDELINES/hunt_new_guidelines.md @@ -5,7 +5,6 @@ Welcome to the `hunting` folder within the `detection-rules` repository! This di ### Documentation and Context - [ ] Detailed description of the Hunt. -- [ ] List any new fields required in ECS/data sources. - [ ] Link related issues or PRs. - [ ] Include references. - [ ] Field Usage: Ensure standardized fields for compatibility across different data environments and sources. @@ -13,19 +12,19 @@ Welcome to the `hunting` folder within the `detection-rules` repository! This di ### Hunt Metadata Checks - [ ] `author`: The name of the individual or organization authoring the rule. -- [ ] `creation_date` matches the date of creation PR initially merged. -- [ ] `min_stack_version` supports the widest stack versions. +- [ ] `uuid`: Unique UUID. - [ ] `name` and `description` are descriptive and typo-free. - [ ] `language`: The query language(s) used in the rule, such as `KQL`, `EQL`, `ES|QL`, `OsQuery`, or `YARA`. - [ ] `query` is inclusive, not overly exclusive, considering performance for diverse environments. - [ ] `integration` aligns with the `index`. Ensure updates if the integration is newly introduced. -- [ ] `setup` includes necessary steps to configure the integration. -- [ ] `note` includes additional information (e.g., Triage and analysis investigation guides, timeline templates). -- [ ] `tags` are relevant to the threat and align with `EXPECTED_HUNT_TAGS` in `definitions.py`. -- [ ] `threat`, `techniques`, and `subtechniques` map to ATT&CK whenever possible. +- [ ] `notes` includes additional information regarding data collected from the hunting query. +- [ ] `mitre` matches appropriate technique and sub-technique IDs that hunting query collect's data for. +- [ ] `references` are valid URL links that include information relevenat to the hunt or threat. +- [ ] `license` ### Testing and Validation -- [ ] Evidence of testing and detecting the expected threat. -- [ ] Check for the existence of coverage to prevent duplication. -- [ ] Generate Markdown: Run `python generate_markdown.py` to update the documentation. +- [ ] Evidence of testing and valid query usage. +- [ ] Markdown Generated: Run `python -m hunting generate-markdown` with specific parameters to ensure a markdown version of the hunting TOML files is created. +- [ ] Index Refreshed: Run `python -m hunting refresh-index` to refresh indexes. +- [ ] Run Unit Tests: Run `pytest tests/test_hunt_data.py` to run unit tests. diff --git a/.github/PULL_REQUEST_GUIDELINES/hunt_tuning_guidelines.md b/.github/PULL_REQUEST_GUIDELINES/hunt_tuning_guidelines.md index 6e1023279..2e4c811ba 100644 --- a/.github/PULL_REQUEST_GUIDELINES/hunt_tuning_guidelines.md +++ b/.github/PULL_REQUEST_GUIDELINES/hunt_tuning_guidelines.md @@ -6,34 +6,25 @@ These guidelines serve as a reminder set of considerations when tuning an existi - [ ] Detailed description of the suggested changes. - [ ] Provide example JSON data or screenshots. -- [ ] Evidence of reducing benign events mistakenly identified as threats (False Positives). -- [ ] Evidence of enhancing detection of true threats that were previously missed (False Negatives). -- [ ] Evidence of optimizing resource consumption and execution time of detection rules (Performance). +- [ ] Evidence of enhancing hunting results by either reducing false-positives or removing false-negatives. - [ ] Evidence of specific environment factors influencing customized hunt tuning (Contextual Tuning). -- [ ] Evidence of improvements by modifying sensitivity (Threshold Adjustments). - [ ] Evidence of refining hunts to better detect deviations from typical behavior (Behavioral Tuning). -- [ ] Evidence of improvements based on time-based patterns (Temporal Tuning). -- [ ] Reasoning for adjusting priority or severity levels of alerts (Severity Tuning). -- [ ] Evidence of improving the quality integrity of data used by hunts (Data Quality). -- [ ] Ensure necessary updates to release documentation and versioning. - [ ] Field Usage: Ensure standardized fields for compatibility across different data environments and sources. ### Hunt Metadata Checks - [ ] `author`: The name of the individual or organization authoring the rule. -- [ ] `updated_date` matches the date of tuning PR merged. -- [ ] `min_stack_version` supports the widest stack versions. - [ ] `name` and `description` are descriptive and typo-free. - [ ] `language`: The query language(s) used in the rule, such as `KQL`, `EQL`, `ES|QL`, `OsQuery`, or `YARA`. - [ ] `query` is inclusive, not overly exclusive. Review to ensure the original intent of the hunt is maintained. - [ ] `integration` aligns with the `index`. Ensure updates if the integration is newly introduced. -- [ ] `setup` includes necessary steps to configure the integration. -- [ ] `note` includes additional information (e.g., Triage and analysis investigation guides, timeline templates). -- [ ] `tags` are relevant to the threat and align with `EXPECTED_HUNT_TAGS` in `definitions.py`. -- [ ] `threat`, `techniques`, and `subtechniques` map to ATT&CK whenever possible. +- [ ] `notes` includes additional information (e.g., Triage and analysis investigation guides, timeline templates). +- [ ] `mitre` matches appropriate technique and sub-technique IDs that hunting query collect's data for. +- [ ] `references` are valid URL links that include information relevenat to the hunt or threat. ### Testing and Validation -- [ ] Generate Markdown: Run `python generate_markdown.py` to update the documentation. -- [ ] Validate the tuned hunt's performance and ensure it does not negatively impact the stack. -- [ ] Ensure the tuned hunt has a low false positive rate. \ No newline at end of file +- [ ] Evidence of testing and valid query usage. +- [ ] Markdown Generated: Run `python -m hunting generate-markdown` with specific parameters to ensure a markdown version of the hunting TOML files is created. +- [ ] Index Refreshed: Run `python -m hunting refresh-index` to refresh indexes. +- [ ] Run Unit Tests: Run `pytest tests/test_hunt_data.py` to run unit tests. \ No newline at end of file diff --git a/.github/paths-labeller.yml b/.github/paths-labeller.yml index 81d91e9f6..6f50c9c1d 100644 --- a/.github/paths-labeller.yml +++ b/.github/paths-labeller.yml @@ -14,6 +14,8 @@ - "kql/**/*.py" - "RTA": - "rta/**/*" +- "Hunting": + - "hunting/**/*" # rules - "bbr": diff --git a/Makefile b/Makefile index 1cfac78f2..bc412d6df 100644 --- a/Makefile +++ b/Makefile @@ -26,6 +26,11 @@ deps: $(VENV) $(PIP) install lib/kibana $(PIP) install lib/kql +.PHONY: hunting-deps +deps: $(VENV) + @echo "Installing all dependencies..." + $(PIP) install .[hunting] + .PHONY: pytest pytest: $(VENV) deps $(PYTHON) -m detection_rules test @@ -53,6 +58,11 @@ test-remote-cli: $(VENV) deps @echo "Executing test_remote_cli script..." @./detection_rules/etc/test_remote_cli.bash +.PHONY: test-hunting-cli +test-remote-cli: $(VENV) hunting-deps + @echo "Executing test_hunting_cli script..." + @./detection_rules/etc/test_hunting_cli.bash + .PHONY: release release: deps @echo "RELEASE: $(app_name)" diff --git a/README.md b/README.md index 427a172c6..4aad23270 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ Detection Rules contains more than just static rule files. This repository also |------------------------------------------------ |------------------------------------------------------------------------------------ | | [`detection_rules/`](detection_rules) | Python module for rule parsing, validating and packaging | | [`etc/`](detection_rules/etc) | Miscellaneous files, such as ECS and Beats schemas | -| [`hunting`](./hunting/) | Root directory where threat hunting queries are stored | +| [`hunting/`](./hunting/) | Root directory where threat hunting package and queries are stored | | [`kibana/`](lib/kibana) | Python library for handling the API calls to Kibana and the Detection Engine | | [`kql/`](lib/kql) | Python library for parsing and validating Kibana Query Language | | [`rta/`](rta) | Red Team Automation code used to emulate attacker techniques, used for rule testing | @@ -78,7 +78,7 @@ Collecting Click==7.0 ... ``` -Note: The `kibana` and `kql` packages are not available on PyPI and must be installed from the `lib` directory. +Note: The `kibana` and `kql` packages are not available on PyPI and must be installed from the `lib` directory. The `hunting` package has optional dependencies to be installed with `pip3 install ".[hunting]`. ```console diff --git a/detection_rules/etc/test_hunting_cli.bash b/detection_rules/etc/test_hunting_cli.bash new file mode 100755 index 000000000..834ea6238 --- /dev/null +++ b/detection_rules/etc/test_hunting_cli.bash @@ -0,0 +1,34 @@ +#!/bin/bash + +# Path to the virtual environment +VENV_PATH="./env/detection-rules-build" + +# Activate the virtual environment +source "$VENV_PATH/bin/activate" + +echo "Running hunting CLI tests..." + +echo "Searching: Search for T1078.004 subtechnique in AWS data source" +python -m hunting search --sub-technique T1078.004 --data-source aws + +echo "Refreshing index" +python -m hunting refresh-index + +echo "Generating Markdown: initial_access_higher_than_average_failed_authentication.toml" +python -m hunting generate-markdown /Users/tdejesus/code/src/detection-rules/hunting/okta/queries/initial_access_higher_than_average_failed_authentication.toml + +echo "Running Query: low_volume_external_network_connections_from_process.toml" +echo "Requires .detection-rules-cfg.json credentials file set." +python -m hunting run-query --file-path /Users/tdejesus/code/src/detection-rules/hunting/linux/queries/low_volume_external_network_connections_from_process.toml --all + +echo "Viewing Hunt: 12526f14-5e35-4f5f-884c-96c6a353a544" +python -m hunting view-hunt --uuid 12526f14-5e35-4f5f-884c-96c6a353a544 --format json + +echo "Generating summary of hunts by integration" +python -m hunting hunt-summary --breakdown integration + +echo "Generating summary of hunts by platform" +python -m hunting hunt-summary --breakdown platform + +echo "Generating summary of hunts by language" +python -m hunting hunt-summary --breakdown language diff --git a/hunting/README.md b/hunting/README.md index aaa161d4b..d2d7f1f20 100644 --- a/hunting/README.md +++ b/hunting/README.md @@ -1,4 +1,4 @@ -# Hunt Queries +# Hunt Queries 🎯 --- @@ -38,36 +38,96 @@ Otherwise, the names do not require the integration, since it is already annotat - **integration**: The specific integration or data source the rule applies to, such as `aws_bedrock.invocation`. - **uuid**: A unique identifier for the rule to maintain version control and tracking. - **name**: A descriptive name for the rule that clearly indicates its purpose. - - **language**: The query language(s) used in the rule, such as `KQL`, `EQL`, `ES|QL`, `OsQuery`, or `YARA`. + - **language**: The query language(s) used in the rule, such as `KQL`, `EQL`, `ES|QL`, `SQL`, or `YARA`. Please note, `SQL` may be used in TOML hunting files, but refers to OSQuery. - **query**: An array of actual queries or analytic expressions written in the appropriate query language that executes the detection logic. - **notes**: An array of strings providing detailed insights into the rationale behind the rule, suggestions for further investigation, and tips on distinguishing false positives from true activity. - **mitre**: Reference to applicable MITRE ATT&CK tactics or techniques that the rule addresses, enhancing the contextual understanding of its security implications. - **references**: Links to external documents, research papers, or websites that provide additional information or validation for the detection logic. -- **Documentation (Optional)**: Include a `README.md` in each subfolder describing the queries and their purposes. This would include a brief description of the new category. - ### Field Usage Use standardized fields where possible to ensure that queries are compatible across different data environments and sources. ### Review and Pull Requests -Follow the standard [contributing guide](../CONTRIBUTING.md). Please remember to use the generate_markdown.py script to update the documentation after adding or updateing queries. +Follow the standard [contributing guide](../CONTRIBUTING.md). Please remember to use the `generate-markdown` command to update the documentation after adding or updating queries. -## Using the Script to Generate Markdown +## Commands -The `generate_markdown.py` script is provided to automate the creation of Markdown files from TOML rule definitions. Here’s how to use it: +The `hunting` folder is an executable package with its own CLI using [click](https://pypi.org/project/click/). All commands can be ran from the root of `detection-rules` repository as such: `python -m hunting COMMAND`. + +- **generate-markdown**: + - This will generate Markdown files for each TOML file specified and update the `index.yml` and `index.md`. + - The `path` parameter is to enable users to specify a single file path of the TOML file, an existing folder (i.e. `aws`) or none, which will generate markdown docs for all hunt queries. + - Rules should be written in TOML and saved under the respective `hunt/*/rules/` directory before running this command. The command will automatically convert them into Markdown and save them in the `docs` directory within the respective category folder. +- **refresh-index**: + - This will load all hunting query TOML files, then overwrite the existing `index.yml`, followed by updating the `index.md` file + - This is important whenever new hunts are created or name, file path or MITRE changes are introduced to existing queries. + - The `search` command relies on the `index.yml` file, so keeping this up-to-date is crucial. +- **search**: + - This command enables users to filter for queries based on MITRE ATT&CK information, more specifically, tactic, technique or sub-technique IDs. The `--tactic`, `--technique`, `--subtechnique` parameters can be used to search for hunting queries that have been tagged with these respective IDs. + - All hunting queries are required to include MITRE mappings. Additionally, `--data-source` parameter can be used with or without MITRE filters to scope to a specific data source (i.e. `python -m hunting search --tactic TA0001 --data-source aws` would show all credential access related hunting queries for AWS) + - More open-ended keyword searches are available via `--keyword` search that can be paired with data source or not to search across a hunting content's name, description, notes and references data. +- **run-query**: **NOTE** - This command requires the `.detection-rules-cfg.yaml` to be populated. Please refer to the [CLI docs](../CLI.md) for optional parameters. + - This command enables users to load a TOML file, select a hunting query and run it against their elasticsearch instance The `--uuid` and `--file-path` parameters can be used to select which hunting query(s) to run. + - Users can select which query to run from the TOML file if multiple are available. + - This command is only meant to identify quickly if matches of the hunting query are found or not. It is recommended to pivot into the UI to either extend the range of the query or investigate matches. + - Only `ES|QL` queries are compatible with this command, but will be determined programmatically by this command if any are available. +- **view-hunt**: + - This command outputs the contents of a hunting file in either JSON or TOML. The `--uuid` and `--file-path` parameters enable users to view by UUID or file path. + - The `--query-only` parameter will only output the queries within the TOML file. +- **hunt-summary**: + - This command outputs a summary of all hunting queries in the repository. The `--breakdown` parameter enables users to see the summary based on integration, language, or platform. + +## Add a Hunt Workflow + +To contribute to the `hunting` folder or add new hunting queries, follow these steps: + +1. **Clone (or fork) and Install Dependencies** + - `git clone git@github.com:elastic/detection-rules.git` to clone the repository + - Setup your own virtual environment if not already established + - `pip install ".[hunting]"` + +2. **Create a TOML File** + - Navigate to the respective folder (e.g., `aws/queries`, `macos/queries`) and create a new TOML file for your query. + - Ensure that the file is named descriptively, reflecting the purpose of the hunt (e.g., `credential_access_detection.toml`). + +3. **Add Relevant and Required Hunting Information** + - Fill out the necessary fields in your TOML file. Be sure to include information such as the author, description, query language, actual queries, MITRE technique mappings, and any notes or references. This ensures the hunt query is complete and provides valuable context for threat hunters. + +4. **Generate the Markdown File** + - Once the TOML file is ready, use the following command to generate the corresponding Markdown file: + ```bash + python -m hunting generate-markdown + ``` + - This will create a Markdown file in the `docs` folder under the respective integration, which can be used for documentation or sharing. + +5. **Refresh the Indexes** + - After generating the Markdown, run the `refresh-indexes` command to update the `index.yml` and `index.md` files: + ```bash + python -m hunting refresh-index + ``` + - This ensures that the new hunt query is reflected in the overall index and is available for searching. + +6. **Open a Pull Request (PR) for Contributions** + - If you're contributing the query to the project, submit a Pull Request (PR) with your changes. Be sure to include a description of your query and any relevant details to facilitate the review process. + +By following this workflow, you can ensure that your hunt queries are properly formatted, documented, and integrated into the Elastic hunting library. -- **Generating Markdown**: Run `python generate_markdown.py` from the root of the `hunting` directory. This will generate Markdown files for each TOML file and update the `index.md` to include links to the new Markdown files. -- **Structure**: Rules should be written in TOML and saved under the respective `hunt/*/rules/` directory. The script will automatically convert them into Markdown and save them in the `docs` directory within the respective category folder. ### Sample Directory Structure Example ```config . -β”œβ”€β”€ README.md -β”œβ”€β”€ generate_markdown.py +β”œβ”€β”€ __init__.py +β”œβ”€β”€ __main__.py +β”œβ”€β”€ definitions.py β”œβ”€β”€ index.md +β”œβ”€β”€ index.yml +β”œβ”€β”€ markdown.py +β”œβ”€β”€ README.md +β”œβ”€β”€ run.py +β”œβ”€β”€ search.py +β”œβ”€β”€ utils.py └── categorical_folder_name - β”œβ”€β”€ README.md β”œβ”€β”€ docs β”‚ └── generated_markdown.md └── rules diff --git a/hunting/__main__.py b/hunting/__main__.py new file mode 100644 index 000000000..4ce320566 --- /dev/null +++ b/hunting/__main__.py @@ -0,0 +1,257 @@ +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0; you may not use this file except in compliance with the Elastic License +# 2.0. + +import json +import textwrap +from collections import Counter +from dataclasses import asdict +from pathlib import Path + +import click +from tabulate import tabulate + +from detection_rules.misc import parse_user_config + +from .definitions import HUNTING_DIR +from .markdown import MarkdownGenerator +from .run import QueryRunner +from .search import QueryIndex +from .utils import (filter_elasticsearch_params, get_hunt_path, load_all_toml, + load_toml, update_index_yml) + + +@click.group() +def hunting(): + """Commands for managing hunting queries and converting TOML to Markdown.""" + pass + + +@hunting.command('generate-markdown') +@click.argument('path', required=False) +def generate_markdown(path: Path = None): + """Convert TOML hunting queries to Markdown format.""" + markdown_generator = MarkdownGenerator(HUNTING_DIR) + + if path: + path = Path(path) + if path.is_file() and path.suffix == '.toml': + click.echo(f"Generating Markdown for single file: {path}") + markdown_generator.process_file(path) + elif (HUNTING_DIR / path).is_dir(): + click.echo(f"Generating Markdown for folder: {path}") + markdown_generator.process_folder(path) + else: + raise ValueError(f"Invalid path provided: {path}") + else: + click.echo("Generating Markdown for all files.") + markdown_generator.process_all_files() + + # After processing, update the index + markdown_generator.update_index_md() + + +@hunting.command('refresh-index') +def refresh_index(): + """Refresh the index.yml file from TOML files and then refresh the index.md file.""" + click.echo("Refreshing the index.yml and index.md files.") + update_index_yml(HUNTING_DIR) + markdown_generator = MarkdownGenerator(HUNTING_DIR) + markdown_generator.update_index_md() + click.echo("Index refresh complete.") + + +@hunting.command('search') +@click.option('--tactic', type=str, default=None, help="Search by MITRE tactic ID (e.g., TA0001)") +@click.option('--technique', type=str, default=None, help="Search by MITRE technique ID (e.g., T1078)") +@click.option('--sub-technique', type=str, default=None, help="Search by MITRE sub-technique ID (e.g., T1078.001)") +@click.option('--data-source', type=str, default=None, help="Filter by data_source like 'aws', 'macos', or 'linux'") +@click.option('--keyword', type=str, default=None, help="Search by keyword in name, description, and notes") +def search_queries(tactic: str, technique: str, sub_technique: str, data_source: str, keyword: str): + """Search for queries based on MITRE tactic, technique, sub-technique, or data_source.""" + + if not any([tactic, technique, sub_technique, data_source, keyword]): + raise click.UsageError("""Please provide at least one filter (tactic, technique, sub-technique, + data_source or keyword) to search queries.""") + + click.echo("Searching for queries based on provided filters...") + + # Create an instance of the QueryIndex class + query_index = QueryIndex(HUNTING_DIR) + + # Filter out None values from the MITRE filter tuple + mitre_filters = tuple(filter(None, (tactic, technique, sub_technique))) + + # Call the search method of QueryIndex with the provided MITRE filters, data_source, and keyword + results = query_index.search(mitre_filter=mitre_filters, data_source=data_source, keyword=keyword) + + if results: + click.secho(f"\nFound {len(results)} matching queries:\n", fg="green", bold=True) + + # Prepare the data for tabulate + table_data = [] + for result in results: + # Customize output to include technique, data_source, and UUID + data_source_str = result['data_source'] + mitre_str = ", ".join(result['mitre']) + uuid = result['uuid'] + table_data.append([result['name'], uuid, result['path'], data_source_str, mitre_str]) + + # Output results using tabulate + table_headers = ["Name", "UUID", "Location", "Data Source", "MITRE"] + click.echo(tabulate(table_data, headers=table_headers, tablefmt="fancy_grid")) + + else: + click.secho("No matching queries found.", fg="red", bold=True) + + +@hunting.command('view-hunt') +@click.option('--uuid', type=str, help="View a specific hunt by UUID.") +@click.option('--path', type=str, help="View a specific hunt by file path.") +@click.option('--format', 'output_format', default='toml', type=click.Choice(['toml', 'json'], case_sensitive=False), + help="Output format (toml or json).") +@click.option('--query-only', is_flag=True, help="Only display the query content.") +def view_hunt(uuid: str, path: str, output_format: str, query_only: bool): + """View a specific hunt by UUID or file path in the specified format (TOML or JSON).""" + + # Get the hunt path or error message + hunt_path, error_message = get_hunt_path(uuid, path) + + if error_message: + raise click.ClickException(error_message) + + # Load the TOML data + hunt = load_toml(hunt_path) + + # Handle query-only option + if query_only: + click.secho("Available queries:", fg="blue", bold=True) + # Format queries for display using tabulate and textwrap + table_data = [(i, textwrap.fill(query, width=120)) for i, query in enumerate(hunt.query)] + table_headers = ["Query"] + click.echo(tabulate(table_data, headers=table_headers, tablefmt="fancy_grid")) + return + + # Output the hunt in the requested format + if output_format == 'toml': + click.echo(hunt_path.read_text()) + elif output_format == 'json': + hunt_dict = asdict(hunt) + click.echo(json.dumps(hunt_dict, indent=4)) + + +@hunting.command('hunt-summary') +@click.option('--breakdown', type=click.Choice(['platform', 'integration', 'language'], + case_sensitive=False), default='platform', + help="Specify how to break down the summary: 'platform', 'integration', or 'language'.") +def hunt_summary(breakdown: str): + """ + Generate a summary of hunt queries, broken down by platform, integration, or language. + """ + click.echo(f"Generating hunt summary broken down by {breakdown}...") + + # Load all hunt queries + all_hunts = load_all_toml(HUNTING_DIR) + + # Use Counter for more concise counting + platform_counter = Counter() + integration_counter = Counter() + language_counter = Counter() + + for hunt, path in all_hunts: + # Get the platform based on the folder name + platform = path.parent.parent.stem + platform_counter[platform] += 1 + + # Count integrations + integration_counter.update(hunt.integration) + + # Count languages, renaming 'SQL' to 'OSQuery' + languages = ['OSQuery' if lang == 'SQL' else lang for lang in hunt.language] + language_counter.update(languages) + + # Prepare and display the table based on the selected breakdown + if breakdown == 'platform': + table_data = [[platform, count] for platform, count in platform_counter.items()] + table_headers = ["Platform (Folder)", "Hunt Count"] + elif breakdown == 'integration': + table_data = [[integration, count] for integration, count in integration_counter.items()] + table_headers = ["Integration", "Hunt Count"] + elif breakdown == 'language': + table_data = [[language, count] for language, count in language_counter.items()] + table_headers = ["Language", "Hunt Count"] + + click.echo(tabulate(table_data, headers=table_headers, tablefmt="fancy_grid")) + + +@hunting.command('run-query') +@click.option('--uuid', help="The UUID of the hunting query to run.") +@click.option('--file-path', help="The file path of the hunting query to run.") +@click.option('--all', 'run_all', is_flag=True, help="Run all eligible queries in the file.") +@click.option('--wait-time', 'wait_time', default=180, help="Time to wait for query completion.") +def run_query(uuid: str, file_path: str, run_all: bool, wait_time: int): + """Run a hunting query by UUID or file path. Only ES|QL queries are supported.""" + + # Get the hunt path or error message + hunt_path, error_message = get_hunt_path(uuid, file_path) + + if error_message: + click.echo(error_message) + return + + # Load the user configuration + config = parse_user_config() + if not config: + click.secho("No configuration found. Please add a `detection-rules-cfg` file.", fg="red", bold=True) + return + + es_config = filter_elasticsearch_params(config) + + # Create a QueryRunner instance + query_runner = QueryRunner(es_config) + + # Load the hunting data + hunting_data = query_runner.load_hunting_file(hunt_path) + + # Display description + wrapped_description = textwrap.fill(hunting_data.description, width=120) + click.secho("\nHunting Description:", fg="blue", bold=True) + click.secho(f"\n{wrapped_description}\n", bold=True) + + # Extract eligible queries + eligible_queries = {i: query for i, query in enumerate(hunting_data.query) if "from" in query} + if not eligible_queries: + click.secho("No eligible queries found in the file.", fg="red", bold=True) + return + + if run_all: + # Run all eligible queries if the --all flag is set + query_runner.run_all_queries(eligible_queries, wait_time) + return + + # Display available queries + click.secho("Available queries:", fg="blue", bold=True) + for i, query in eligible_queries.items(): + click.secho(f"\nQuery {i + 1}:", fg="green", bold=True) + click.echo(query_runner._format_query(query)) + click.secho("\n" + "-" * 120, fg="yellow") + + # Handle query selection + while True: + try: + query_number = click.prompt("Enter the query number", type=int) + if query_number - 1 in eligible_queries: + selected_query = eligible_queries[query_number - 1] + break + else: + click.secho(f"Invalid query number: {query_number}. Please try again.", fg="yellow") + except ValueError: + click.secho("Please enter a valid number.", fg="yellow") + + # Run the selected query + query_runner.run_individual_query(selected_query, wait_time) + + +if __name__ == "__main__": + hunting() diff --git a/hunting/aws/docs/ec2_discovery_multi_region_get_service_quota_calls.md b/hunting/aws/docs/ec2_discovery_multi_region_get_service_quota_calls.md deleted file mode 100644 index 6a98a93cd..000000000 --- a/hunting/aws/docs/ec2_discovery_multi_region_get_service_quota_calls.md +++ /dev/null @@ -1,58 +0,0 @@ -# High Frequency of EC2 Multi-Region `GetServiceQuota` API Calls - ---- - -## Metadata - -- **Author:** Elastic -- **Description:** This hunting query identifies when a single AWS resource is making `GetServiceQuota` API calls for the EC2 service quota L-1216C47A in more than 10 regions within a 30-second window. Quota code L-1216C47A represents on-demand instances which are used by adversaries to deploy malware and mine cryptocurrency. This could indicate a potential threat actor attempting to discover the AWS infrastructure across multiple regions using compromised credentials or a compromised instance. - -- **UUID:** `7a083b24-6482-11ef-8a8f-f661ea17fbcc` -- **Integration:** [aws.cloudtrail](https://docs.elastic.co/integrations/aws/cloudtrail) -- **Language:** `[ES|QL]` -- **Source File:** [High Frequency of EC2 Multi-Region `GetServiceQuota` API Calls](../queries/ec2_discovery_multi_region_get_service_quota_calls.toml) - -## Query - -```sql -from logs-aws.cloudtrail-* -| where @timestamp > now() - 7 day - -// filter for GetServiceQuota API calls -| where event.dataset == "aws.cloudtrail" and event.action == "GetServiceQuota" - -// truncate the timestamp to a 30-second window -| eval target_time_window = DATE_TRUNC(30 seconds, @timestamp) - -// pre-process the request parameters to extract the service code and quota code -| dissect aws.cloudtrail.request_parameters "{%{?service_code_key}=%{service_code}, %{?quota_code_key}=%{quota_code}}" - -// filter for EC2 service quota L-1216C47A (vCPU on-demand instances) -| where service_code == "ec2" and quota_code == "L-1216C47A" - -// count the number of unique regions and total API calls within the 30-second window -| stats region_count = count_distinct(cloud.region), window_count = count(*) by target_time_window, aws.cloudtrail.user_identity.arn - -// filter for resources making DescribeInstances API calls in more than 10 regions within the 30-second window -| where region_count >= 10 and window_count >= 10 - -// sort the results by time windows in descending order -| sort target_time_window desc -``` - -## Notes - -- Use the `aws.cloudtrail.user_identity.arn` field to identify the user making the requests and their role permissions -- Use the `cloud.region` field to identify the regions where the `GetServiceQuota` API calls were made -- Review Elastic Defend alerts for endpoint related activity to identify potential malware or cryptocurrency mining activity -- If a valid account compromise is suspected, review source.* fields for the IP address and geographical location of the request and compare with the user's typical behavior -- Query for `RunInstances` API calls to determine if new instances were launched using the on-demand instances -- If new instances were launched, review the instance metadata and user data scripts for malicious content - -## MITRE ATT&CK Techniques - -- [T1580](https://attack.mitre.org/techniques/T1580) - -## License - -- `Elastic License v2` diff --git a/hunting/aws/docs/s3_public_bucket_rapid_object_access_attempts.md b/hunting/aws/docs/s3_public_bucket_rapid_object_access_attempts.md index d5d71df1e..770de959e 100644 --- a/hunting/aws/docs/s3_public_bucket_rapid_object_access_attempts.md +++ b/hunting/aws/docs/s3_public_bucket_rapid_object_access_attempts.md @@ -7,7 +7,7 @@ - **Author:** Elastic - **Description:** This hunting query identifies when an anonymous user, outside of the known AWS IP ranges, makes multiple `GetObject` requests to a public S3 bucket. Rapid access to objects in a public S3 bucket may indicate an adversary attempting to exfiltrate data or perform reconnaissance on the bucket contents. -- **UUID:** `ef244ca0-5e32-11ef-a8d3-f661ea17fbce` +- **UUID:** `ef579900-75ef-11ef-b47f-f661ea17fbcc` - **Integration:** [aws.cloudtrail](https://docs.elastic.co/integrations/aws/cloudtrail) - **Language:** `[ES|QL]` - **Source File:** [S3 Public Bucket Rapid Object Access Attempts](../queries/s3_public_bucket_rapid_object_access_attempts.toml) diff --git a/hunting/aws/queries/s3_public_bucket_rapid_object_access_attempts.toml b/hunting/aws/queries/s3_public_bucket_rapid_object_access_attempts.toml index edb77a7c8..afed607ab 100644 --- a/hunting/aws/queries/s3_public_bucket_rapid_object_access_attempts.toml +++ b/hunting/aws/queries/s3_public_bucket_rapid_object_access_attempts.toml @@ -4,7 +4,7 @@ description = """ This hunting query identifies when an anonymous user, outside of the known AWS IP ranges, makes multiple `GetObject` requests to a public S3 bucket. Rapid access to objects in a public S3 bucket may indicate an adversary attempting to exfiltrate data or perform reconnaissance on the bucket contents. """ integration = ["aws.cloudtrail"] -uuid = "ef244ca0-5e32-11ef-a8d3-f661ea17fbce" +uuid = "ef579900-75ef-11ef-b47f-f661ea17fbcc" name = "S3 Public Bucket Rapid Object Access Attempts" language = ["ES|QL"] license = "Elastic License v2" diff --git a/hunting/definitions.py b/hunting/definitions.py new file mode 100644 index 000000000..990aa839d --- /dev/null +++ b/hunting/definitions.py @@ -0,0 +1,36 @@ +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0; you may not use this file except in compliance with the Elastic License +# 2.0. + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Optional + +# Define the hunting directory path +HUNTING_DIR = Path(__file__).parent + +# URLs for MITRE and Elastic documentation +ATLAS_URL = "https://atlas.mitre.org/techniques/" +ATTACK_URL = "https://attack.mitre.org/techniques/" + +# Static mapping for specific integrations +STATIC_INTEGRATION_LINK_MAP = { + 'aws_bedrock.invocation': 'aws_bedrock' +} + + +@dataclass +class Hunt: + """Dataclass to represent a hunt.""" + author: str + description: str + integration: list[str] + uuid: str + name: str + language: list[str] + license: str + query: list[str] + notes: Optional[list[str]] = field(default_factory=list) + mitre: list[str] = field(default_factory=list) + references: Optional[list[str]] = field(default_factory=list) diff --git a/hunting/generate_markdown.py b/hunting/generate_markdown.py deleted file mode 100644 index 9db91cced..000000000 --- a/hunting/generate_markdown.py +++ /dev/null @@ -1,140 +0,0 @@ -# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0; you may not use this file except in compliance with the Elastic License -# 2.0. - -"""Lightweight builtin toml-markdown converter.""" - -import tomllib -import urllib3 -from dataclasses import dataclass, field -from pathlib import Path -from typing import List, Optional - -HUNTING_DIR = Path(__file__).parent -ATLAS_URL = "https://atlas.mitre.org/techniques/" -ATTACK_URL = "https://attack.mitre.org/techniques/" - -# the standard link takes `integration.package` and converts the link to `integration/package`, however, there are -# some exceptions such as `aws_bedrock.invocation` which should be linked to `aws_bedrock` instead -# https://docs.elastic.co/integrations/aws_bedrock -STATIC_INTEGRATION_LINK_MAP = { - 'aws_bedrock.invocation': 'aws_bedrock' -} - - -@dataclass -class Hunt: - """Dataclass to represent a hunt.""" - - author: str - description: str - integration: list[str] - uuid: str - name: str - language: list[str] - license: str - query: list[str] - notes: Optional[list[str]] = field(default_factory=list) - mitre: Optional[list[str]] = field(default_factory=list) - references: Optional[list[str]] = field(default_factory=list) - - -def load_toml(contents: str) -> Hunt: - """Load and validate TOML content as Hunt dataclass.""" - toml_dict = tomllib.loads(contents) - return Hunt(**toml_dict["hunt"]) - - -def load_all_toml(base_path: Path) -> List[tuple[Hunt, Path]]: - """Load all TOML files from the directory and return a list of Hunt configurations and their paths.""" - hunts = [] - for toml_file in base_path.rglob("*.toml"): - hunt_config = load_toml(toml_file.read_text(encoding="utf-8")) - hunts.append((hunt_config, toml_file)) - return hunts - - -def validate_link(link: str): - """Validate and return the link.""" - http = urllib3.PoolManager() - response = http.request('GET', link) - if response.status != 200: - raise ValueError(f"Invalid link: {link}") - - -def generate_integration_links(integrations: list[str]) -> list[str]: - base_url = 'https://docs.elastic.co/integrations' - generated = [] - for integration in integrations: - if integration in STATIC_INTEGRATION_LINK_MAP: - link_str = STATIC_INTEGRATION_LINK_MAP[integration] - else: - link_str = integration.replace('.', '/') - link = f'{base_url}/{link_str}' - validate_link(link) - generated.append(f'[{integration}]({link})') - - return generated - - -def convert_toml_to_markdown(hunt_config: Hunt, file_path: Path) -> str: - """Convert Hunt to Markdown format.""" - markdown = f"# {hunt_config.name}\n\n---\n\n" - markdown += "## Metadata\n\n" - markdown += f"- **Author:** {hunt_config.author}\n" - markdown += f"- **Description:** {hunt_config.description}\n" - markdown += f"- **UUID:** `{hunt_config.uuid}`\n" - markdown += f"- **Integration:** {', '.join(generate_integration_links(hunt_config.integration))}\n" - markdown += f"- **Language:** `{hunt_config.language}`\n".replace("'", "").replace('"', "") - markdown += f"- **Source File:** [{hunt_config.name}]({(Path('../queries') / file_path.name).as_posix()})\n" - markdown += "\n## Query\n\n" - for query in hunt_config.query: - markdown += f"```sql\n{query}```\n\n" - - if hunt_config.notes: - markdown += "## Notes\n\n" + "\n".join(f"- {note}" for note in hunt_config.notes) - if hunt_config.mitre: - markdown += "\n\n## MITRE ATT&CK Techniques\n\n" + "\n".join( - f"- [{tech}]({ATLAS_URL if tech.startswith('AML') else ATTACK_URL}" - f"{tech.replace('.', '/') if tech.startswith('T') else tech})" - for tech in hunt_config.mitre - ) - if hunt_config.references: - markdown += "\n\n## References\n\n" + "\n".join(f"- {ref}" for ref in hunt_config.references) - - markdown += f"\n\n## License\n\n- `{hunt_config.license}`\n" - return markdown - - -def process_toml_files(base_path: Path) -> None: - """Process all TOML files in the directory recursively and convert them to Markdown.""" - hunts = load_all_toml(base_path) - index_content = "# List of Available Queries\n\nHere are the queries currently available:" - directories = {} - - for hunt_config, toml_file in hunts: - markdown_content = convert_toml_to_markdown(hunt_config, toml_file) - markdown_path = toml_file.parent.parent / "docs" / f"{toml_file.stem}.md" - markdown_path.parent.mkdir(parents=True, exist_ok=True) - markdown_path.write_text(markdown_content, encoding="utf-8") - print(f"Markdown generated: {markdown_path}") - relative_path = markdown_path.relative_to(base_path) - folder_name = toml_file.parent.parent.name - directories.setdefault(folder_name, []).append((relative_path, hunt_config.name, hunt_config.language)) - - # Build index content - for folder, files in sorted(directories.items()): - index_content += f"\n\n## {folder}\n" - for file_path, rule_name, language in sorted(files): - index_path = f"./{file_path.as_posix()}" - index_content += f"- [{rule_name}]({index_path}) ({', '.join(language)})\n" - - # Write the index file at the base directory level - index_path = base_path / "index.md" - index_path.write_text(index_content, encoding="utf-8") - print(f"Index Markdown generated at: {index_path}") - - -if __name__ == "__main__": - process_toml_files(HUNTING_DIR) diff --git a/hunting/index.md b/hunting/index.md index 8962a3f40..732ce4774 100644 --- a/hunting/index.md +++ b/hunting/index.md @@ -2,72 +2,59 @@ Here are the queries currently available: + ## aws -- [High Frequency of EC2 Multi-Region `DescribeInstances` API Calls](./aws/docs/ec2_discovery_multi_region_describe_instance_calls.md) (ES|QL) -- [High EC2 Instance Deployment Count Attempts by Single User or Role](./aws/docs/ec2_high_instance_deployment_count_attempts.md) (ES|QL) - [EC2 Modify Instance Attribute User Data](./aws/docs/ec2_modify_instance_attribute_user_data.md) (ES|QL) - [EC2 Suspicious Get User Password Request](./aws/docs/ec2_suspicious_get_user_password_request.md) (ES|QL) +- [High EC2 Instance Deployment Count Attempts by Single User or Role](./aws/docs/ec2_high_instance_deployment_count_attempts.md) (ES|QL) +- [High Frequency of EC2 Multi-Region `DescribeInstances` API Calls](./aws/docs/ec2_discovery_multi_region_describe_instance_calls.md) (ES|QL) +- [High Frequency of Service Quotas Multi-Region `GetServiceQuota` API Calls](./aws/docs/servicequotas_discovery_multi_region_get_service_quota_calls.md) (ES|QL) - [IAM Assume Role Creation with Attached Policy](./aws/docs/iam_assume_role_creation_with_attached_policy.md) (ES|QL) - [IAM User Activity with No MFA Session](./aws/docs/iam_user_activity_with_no_mfa_session.md) (ES|QL) -- [User Creation with Administrator Policy Assigned](./aws/docs/iam_user_creation_with_administrator_policy_assigned.md) (ES|QL) - [Lambda Add Permissions for Write Actions to Function](./aws/docs/lambda_add_permissions_for_write_actions_to_function.md) (ES|QL) - [Multiple Service Logging Deleted or Stopped](./aws/docs/multiple_service_logging_deleted_or_stopped.md) (ES|QL) - [S3 Public Bucket Rapid Object Access Attempts](./aws/docs/s3_public_bucket_rapid_object_access_attempts.md) (ES|QL) -- [Secrets Manager High Frequency of Programmatic GetSecretValue API Calls](./aws/docs/secretsmanager_high_frequency_get_secret_value.md) (ES|QL) -- [High Frequency of Service Quotas Multi-Region `GetServiceQuota` API Calls](./aws/docs/servicequotas_discovery_multi_region_get_service_quota_calls.md) (ES|QL) -- [Signin Single Factor Console Login via Federated Session](./aws/docs/signin_single_factor_console_login_via_federated_session.md) (ES|QL) - [SSM Rare SendCommand Code Execution by EC2 Instance](./aws/docs/ssm_rare_sendcommand_code_execution.md) (ES|QL) - [SSM SendCommand API Used by EC2 Instance](./aws/docs/ssm_sendcommand_api_used_by_ec2_instance.md) (ES|QL) - [SSM Start Remote Session to EC2 Instance](./aws/docs/ssm_start_remote_session_to_ec2_instance.md) (ES|QL) - [STS Suspicious Federated Temporary Credential Request](./aws/docs/sts_suspicious_federated_temporary_credential_request.md) (ES|QL) - - -## docs -- [Rapid MFA Deny Push Notifications (MFA Bombing)](./okta/docs/docs/credential_access_mfa_bombing_push_notications.md) (ES|QL) -- [Rapid Reset Password Requests for Different Users](./okta/docs/docs/credential_access_rapid_reset_password_requests_for_different_users.md) (ES|QL) -- [Failed OAuth Access Token Retrieval via Public Client App](./okta/docs/docs/defense_evasion_failed_oauth_access_token_retrieval_via_public_client_app.md) (ES|QL) -- [Multiple Application SSO Authentication from the Same Source](./okta/docs/docs/defense_evasion_multiple_application_sso_authentication_repeat_source.md) (ES|QL) -- [OAuth Access Token Granted for Public Client App from Multiple Client Addresses](./okta/docs/docs/defense_evasion_multiple_client_sources_reported_for_oauth_access_tokens_granted.md) (ES|QL) -- [Rare Occurrence of OAuth Access Token Granted to Public Client App](./okta/docs/docs/defense_evasion_rare_oauth_access_token_granted_by_application.md) (ES|QL) -- [Identify High Average of Failed Daily Authentication Attempts](./okta/docs/docs/initial_access_higher_than_average_failed_authentication.md) (ES|QL) -- [Successful Impossible Travel Sign-On Events](./okta/docs/docs/initial_access_impossible_travel_sign_on.md) (ES|QL) -- [Password Spraying from Repeat Source](./okta/docs/docs/initial_access_password_spraying_from_repeat_source.md) (ES|QL) -- [Multi-Factor Authentication (MFA) Push Notification Bombing](./okta/docs/docs/persistence_multi_factor_push_notification_bombing.md) (ES|QL) -- [Rare Occurrence of Domain with User Authentication Events](./okta/docs/docs/persistence_rare_domain_with_user_authentication.md) (ES|QL) +- [Secrets Manager High Frequency of Programmatic GetSecretValue API Calls](./aws/docs/secretsmanager_high_frequency_get_secret_value.md) (ES|QL) +- [Signin Single Factor Console Login via Federated Session](./aws/docs/signin_single_factor_console_login_via_federated_session.md) (ES|QL) +- [User Creation with Administrator Policy Assigned](./aws/docs/iam_user_creation_with_administrator_policy_assigned.md) (ES|QL) ## linux -- [Network Connections with Low Occurrence Frequency for Unique Agent ID](./linux/docs/command_and_control_via_network_connections_with_low_occurrence_frequency_for_unique_agents.md) (ES|QL) -- [Unusual File Downloads from Source Addresses](./linux/docs/command_and_control_via_unusual_file_downloads_from_source_addresses.md) (ES|QL) - [Defense Evasion via Capitalized Process Execution](./linux/docs/defense_evasion_via_capitalized_process_execution.md) (ES|QL) -- [Hidden Process Execution](./linux/docs/defense_evasion_via_hidden_process_execution.md) (ES|QL) -- [Potential Defense Evasion via Multi-Dot Process Execution](./linux/docs/defense_evasion_via_multi_dot_process_execution.md) (ES|QL) +- [Drivers Load with Low Occurrence Frequency](./linux/docs/persistence_via_driver_load_with_low_occurrence_frequency.md) (ES|QL) - [Excessive SSH Network Activity to Unique Destinations](./linux/docs/excessive_ssh_network_activity_unique_destinations.md) (ES|QL) -- [Uncommon Process Execution from Suspicious Directory](./linux/docs/execution_uncommon_process_execution_from_suspicious_directory.md) (ES|QL) +- [Git Hook/Pager Persistence](./linux/docs/persistence_via_git_hook_pager.md) (ES|QL) +- [Hidden Process Execution](./linux/docs/defense_evasion_via_hidden_process_execution.md) (ES|QL) - [Logon Activity by Source IP](./linux/docs/login_activity_by_source_address.md) (ES|QL) - [Low Volume External Network Connections from Process by Unique Agent](./linux/docs/low_volume_external_network_connections_from_process.md) (ES|QL) - [Low Volume GTFOBins External Network Connections](./linux/docs/low_volume_gtfobins_external_network_connections.md) (ES|QL) - [Low Volume Modifications to Critical System Binaries by Unique Host](./linux/docs/low_volume_modifications_to_critical_system_binaries.md) (ES|QL) - [Low Volume Process Injection-Related Syscalls by Process Executable](./linux/docs/low_volume_process_injection_syscalls_by_executable.md) (ES|QL) -- [Persistence Through Reverse/Bind Shells](./linux/docs/persistence_reverse_bind_shells.md) (SQL) -- [Persistence via Cron](./linux/docs/persistence_via_cron.md) (ES|QL, SQL) -- [Drivers Load with Low Occurrence Frequency](./linux/docs/persistence_via_driver_load_with_low_occurrence_frequency.md) (ES|QL) -- [Git Hook/Pager Persistence](./linux/docs/persistence_via_git_hook_pager.md) (ES|QL, SQL) -- [Persistence via Message-of-the-Day](./linux/docs/persistence_via_message_of_the_day.md) (ES|QL, SQL) -- [Persistence via Package Manager](./linux/docs/persistence_via_package_manager.md) (ES|QL, SQL) -- [Persistence via rc.local/rc.common](./linux/docs/persistence_via_rc_local.md) (ES|QL, SQL) -- [Shell Modification Persistence](./linux/docs/persistence_via_shell_modification_persistence.md) (ES|QL, SQL) -- [Persistence via SSH Configurations and/or Keys](./linux/docs/persistence_via_ssh_configurations_and_keys.md) (SQL) -- [Persistence via Systemd (Timers)](./linux/docs/persistence_via_systemd_timers.md) (ES|QL, SQL) -- [Persistence via System V Init](./linux/docs/persistence_via_sysv_init.md) (ES|QL, SQL) -- [Persistence via Udev](./linux/docs/persistence_via_udev.md) (ES|QL, SQL) -- [Unusual System Binary Parent (Potential System Binary Hijacking Attempt)](./linux/docs/persistence_via_unusual_system_binary_parent.md) (ES|QL) -- [Privilege Escalation/Persistence via User/Group Creation and/or Modification](./linux/docs/persistence_via_user_group_creation_modification.md) (SQL) -- [XDG Persistence](./linux/docs/persistence_via_xdg_autostart_modifications.md) (ES|QL, SQL) -- [Privilege Escalation Identification via Existing Sudoers File](./linux/docs/privilege_escalation_via_existing_sudoers.md) (SQL) +- [Network Connections with Low Occurrence Frequency for Unique Agent ID](./linux/docs/command_and_control_via_network_connections_with_low_occurrence_frequency_for_unique_agents.md) (ES|QL) +- [OSQuery SUID Hunting](./linux/docs/privilege_escalation_via_suid_binaries.md) (ES|QL) +- [Persistence Through Reverse/Bind Shells](./linux/docs/persistence_reverse_bind_shells.md) (ES|QL) +- [Persistence via Cron](./linux/docs/persistence_via_cron.md) (ES|QL) +- [Persistence via Message-of-the-Day](./linux/docs/persistence_via_message_of_the_day.md) (ES|QL) +- [Persistence via Package Manager](./linux/docs/persistence_via_package_manager.md) (ES|QL) +- [Persistence via SSH Configurations and/or Keys](./linux/docs/persistence_via_ssh_configurations_and_keys.md) (ES|QL) +- [Persistence via System V Init](./linux/docs/persistence_via_sysv_init.md) (ES|QL) +- [Persistence via Systemd (Timers)](./linux/docs/persistence_via_systemd_timers.md) (ES|QL) +- [Persistence via Udev](./linux/docs/persistence_via_udev.md) (ES|QL) +- [Persistence via rc.local/rc.common](./linux/docs/persistence_via_rc_local.md) (ES|QL) +- [Potential Defense Evasion via Multi-Dot Process Execution](./linux/docs/defense_evasion_via_multi_dot_process_execution.md) (ES|QL) +- [Privilege Escalation Identification via Existing Sudoers File](./linux/docs/privilege_escalation_via_existing_sudoers.md) (ES|QL) +- [Privilege Escalation/Persistence via User/Group Creation and/or Modification](./linux/docs/persistence_via_user_group_creation_modification.md) (ES|QL) - [Process Capability Hunting](./linux/docs/privilege_escalation_via_process_capabilities.md) (ES|QL) - [Segmentation Fault & Potential Buffer Overflow Hunting](./linux/docs/privilege_escalation_via_segmentation_fault_and_buffer_overflow.md) (ES|QL) -- [OSQuery SUID Hunting](./linux/docs/privilege_escalation_via_suid_binaries.md) (SQL) +- [Shell Modification Persistence](./linux/docs/persistence_via_shell_modification_persistence.md) (ES|QL) +- [Uncommon Process Execution from Suspicious Directory](./linux/docs/execution_uncommon_process_execution_from_suspicious_directory.md) (ES|QL) +- [Unusual File Downloads from Source Addresses](./linux/docs/command_and_control_via_unusual_file_downloads_from_source_addresses.md) (ES|QL) +- [Unusual System Binary Parent (Potential System Binary Hijacking Attempt)](./linux/docs/persistence_via_unusual_system_binary_parent.md) (ES|QL) +- [XDG Persistence](./linux/docs/persistence_via_xdg_autostart_modifications.md) (ES|QL) ## llm @@ -82,34 +69,48 @@ Here are the queries currently available: - [Suspicious Network Connections by Unsigned Mach-O](./macos/docs/suspicious_network_connections_by_unsigned_macho.md) (ES|QL) +## okta +- [Failed OAuth Access Token Retrieval via Public Client App](./okta/docs/defense_evasion_failed_oauth_access_token_retrieval_via_public_client_app.md) (ES|QL) +- [Identify High Average of Failed Daily Authentication Attempts](./okta/docs/initial_access_higher_than_average_failed_authentication.md) (ES|QL) +- [Multi-Factor Authentication (MFA) Push Notification Bombing](./okta/docs/persistence_multi_factor_push_notification_bombing.md) (ES|QL) +- [Multiple Application SSO Authentication from the Same Source](./okta/docs/defense_evasion_multiple_application_sso_authentication_repeat_source.md) (ES|QL) +- [OAuth Access Token Granted for Public Client App from Multiple Client Addresses](./okta/docs/defense_evasion_multiple_client_sources_reported_for_oauth_access_tokens_granted.md) (ES|QL) +- [Password Spraying from Repeat Source](./okta/docs/initial_access_password_spraying_from_repeat_source.md) (ES|QL) +- [Rapid MFA Deny Push Notifications (MFA Bombing)](./okta/docs/credential_access_mfa_bombing_push_notications.md) (ES|QL) +- [Rapid Reset Password Requests for Different Users](./okta/docs/credential_access_rapid_reset_password_requests_for_different_users.md) (ES|QL) +- [Rare Occurrence of Domain with User Authentication Events](./okta/docs/persistence_rare_domain_with_user_authentication.md) (ES|QL) +- [Rare Occurrence of OAuth Access Token Granted to Public Client App](./okta/docs/defense_evasion_rare_oauth_access_token_granted_by_application.md) (ES|QL) +- [Successful Impossible Travel Sign-On Events](./okta/docs/initial_access_impossible_travel_sign_on.md) (ES|QL) + + ## windows -- [Low Occurrence Rate of CreateRemoteThread by Source Process](./windows/docs/createremotethread_by_source_process_with_low_occurrence.md) (ES|QL) - [DLL Hijack via Masquerading as Microsoft Native Libraries](./windows/docs/detect_dll_hijack_via_masquerading_as_microsoft_native_libraries.md) (ES|QL) -- [Masquerading Attempts as Native Windows Binaries](./windows/docs/detect_masquerading_attempts_as_native_windows_binaries.md) (ES|QL) -- [Rare DLL Side-Loading by Occurrence](./windows/docs/detect_rare_dll_sideload_by_occurrence.md) (ES|QL) -- [Rare LSASS Process Access Attempts](./windows/docs/detect_rare_lsass_process_access_attempts.md) (ES|QL) - [DNS Queries via LOLBins with Low Occurence Frequency](./windows/docs/domain_names_queried_via_lolbins_and_with_low_occurence_frequency.md) (ES|QL) -- [Low Occurrence of Drivers Loaded on Unique Hosts](./windows/docs/drivers_load_with_low_occurrence_frequency.md) (ES|QL) +- [Egress Network Connections with Total Bytes Greater than Threshold](./windows/docs/potential_exfiltration_by_process_total_egress_bytes.md) (ES|QL) - [Excessive RDP Network Activity by Host and User](./windows/docs/excessive_rdp_network_activity_by_source_host_and_user.md) (ES|QL) - [Excessive SMB Network Activity by Process ID](./windows/docs/excessive_smb_network_activity_by_process_id.md) (ES|QL) - [Executable File Creation by an Unusual Microsoft Binary](./windows/docs/executable_file_creation_by_an_unusual_microsoft_binary.md) (ES|QL) -- [Frequency of Process Execution via Network Logon by Source Address](./windows/docs/execution_via_network_logon_by_occurrence_frequency_by_top_source_ip.md) (ES|QL) - [Execution via Remote Services by Client Address](./windows/docs/execution_via_remote_services_by_client_address.md) (ES|QL) -- [Startup Execution with Low Occurrence Frequency by Unique Host](./windows/docs/execution_via_startup_with_low_occurrence_frequency.md) (ES|QL) +- [Frequency of Process Execution via Network Logon by Source Address](./windows/docs/execution_via_network_logon_by_occurrence_frequency_by_top_source_ip.md) (ES|QL) +- [High Count of Network Connection Over Extended Period by Process](./windows/docs/high_count_of_network_connection_over_extended_period_by_process.md) (ES|QL) +- [Libraries Loaded by svchost with Low Occurrence Frequency](./windows/docs/libraries_loaded_by_svchost_with_low_occurrence_frequency.md) (ES|QL) - [Low Frequency of Process Execution via WMI by Unique Agent](./windows/docs/execution_via_windows_management_instrumentation_by_occurrence_frequency_by_unique_agent.md) (ES|QL) - [Low Frequency of Process Execution via Windows Scheduled Task by Unique Agent](./windows/docs/execution_via_windows_scheduled_task_with_low_occurrence_frequency.md) (ES|QL) - [Low Occurence of Process Execution via Windows Services with Unique Agent](./windows/docs/execution_via_windows_services_with_low_occurrence_frequency.md) (ES|QL) -- [High Count of Network Connection Over Extended Period by Process](./windows/docs/high_count_of_network_connection_over_extended_period_by_process.md) (ES|QL) -- [Libraries Loaded by svchost with Low Occurrence Frequency](./windows/docs/libraries_loaded_by_svchost_with_low_occurrence_frequency.md) (ES|QL) +- [Low Occurrence Rate of CreateRemoteThread by Source Process](./windows/docs/createremotethread_by_source_process_with_low_occurrence.md) (ES|QL) +- [Low Occurrence of Drivers Loaded on Unique Hosts](./windows/docs/drivers_load_with_low_occurrence_frequency.md) (ES|QL) +- [Masquerading Attempts as Native Windows Binaries](./windows/docs/detect_masquerading_attempts_as_native_windows_binaries.md) (ES|QL) - [Microsoft Office Child Processes with Low Occurrence Frequency by Unique Agent](./windows/docs/microsoft_office_child_processes_with_low_occurrence_frequency.md) (ES|QL) - [Network Discovery via Sensitive Ports by Unusual Process](./windows/docs/network_discovery_via_sensitive_ports_by_unusual_process.md) (ES|QL) - [PE File Transfer via SMB_Admin Shares by Agent or User](./windows/docs/pe_file_transfer_via_smb_admin_shares_by_agent.md) (ES|QL) - [Persistence via Run Key with Low Occurrence Frequency](./windows/docs/persistence_via_run_key_with_low_occurrence_frequency.md) (ES|QL) - [Persistence via Startup with Low Occurrence Frequency by Unique Host](./windows/docs/persistence_via_startup_with_low_occurrence_frequency.md) (ES|QL) -- [Egress Network Connections with Total Bytes Greater than Threshold](./windows/docs/potential_exfiltration_by_process_total_egress_bytes.md) (ES|QL) +- [Rare DLL Side-Loading by Occurrence](./windows/docs/detect_rare_dll_sideload_by_occurrence.md) (ES|QL) +- [Rare LSASS Process Access Attempts](./windows/docs/detect_rare_lsass_process_access_attempts.md) (ES|QL) - [Rundll32 Execution Aggregated by Command Line](./windows/docs/rundll32_execution_aggregated_by_cmdline.md) (ES|QL) -- [Scheduled tasks Creation by Action via Registry](./windows/docs/scheduled_task_creation_by_action_via_registry.md) (ES|QL) - [Scheduled Tasks Creation for Unique Hosts by Task Command](./windows/docs/scheduled_tasks_creation_for_unique_hosts_by_task_command.md) (ES|QL) +- [Scheduled tasks Creation by Action via Registry](./windows/docs/scheduled_task_creation_by_action_via_registry.md) (ES|QL) +- [Startup Execution with Low Occurrence Frequency by Unique Host](./windows/docs/execution_via_startup_with_low_occurrence_frequency.md) (ES|QL) - [Suspicious Base64 Encoded Powershell Command](./windows/docs/suspicious_base64_encoded_powershell_commands.md) (ES|QL) - [Suspicious DNS TXT Record Lookups by Process](./windows/docs/suspicious_dns_txt_record_lookups_by_process.md) (ES|QL) - [Unique Windows Services Creation by Service File Name](./windows/docs/unique_windows_services_creation_by_servicefilename.md) (ES|QL) diff --git a/hunting/index.yml b/hunting/index.yml new file mode 100644 index 000000000..867baabb0 --- /dev/null +++ b/hunting/index.yml @@ -0,0 +1,561 @@ +llm: + 11e33a8f-805b-4394-bee0-08ae8d78b025: + name: AWS Bedrock LLM Sensitive Content Refusals + path: ./llm/queries/aws_bedrock_sensitive_content_refusal_detection.toml + mitre: + - AML.T0051 + 00023411-192e-4472-90aa-da7562bc3f2a: + name: AWS Bedrock LLM Denial-of-Service or Resource Exhaustion + path: ./llm/queries/aws_bedrock_dos_resource_exhaustion_detection.toml + mitre: + - AML.T0034 + 131e5887-463a-46a1-a44e-b96361bc6cbc: + name: AWS Bedrock LLM Ignore Previous Prompt Detection + path: ./llm/queries/aws_bedrock_ignore_previous_prompt_detection.toml + mitre: + - AML.T0051.000 + 991b55c3-6327-4af6-8e0c-5d4870748369: + name: AWS Bedrock LLM Latency Anomalies + path: ./llm/queries/aws_bedrock_latency_anomalies_detection.toml + mitre: + - AML.T0029 +macos: + dc04d70a-80aa-4c3f-ad02-2b18d54af6d4: + name: Suspicious Network Connections by Unsigned Mach-O + path: ./macos/queries/suspicious_network_connections_by_unsigned_macho.toml + mitre: + - T1071 + 69fc4f40-8fb1-4652-99b7-52755cd370fe: + name: Low Occurrence of Suspicious Launch Agent or Launch Daemon + path: ./macos/queries/persistence_via_suspicious_launch_agent_or_launch_daemon_with_low_occurrence.toml + mitre: + - T1547 + - T1547.011 + - T1543 + - T1543.001 + - T1543.004 +linux: + ecd84bc7-32ae-474b-93a8-d1d9736c3464: + name: Network Connections with Low Occurrence Frequency for Unique Agent ID + path: ./linux/queries/command_and_control_via_network_connections_with_low_occurrence_frequency_for_unique_agents.toml + mitre: + - T1071.001 + - T1071.004 + 2db642d2-621a-4183-88b5-b2659dc2c940: + name: OSQuery SUID Hunting + path: ./linux/queries/privilege_escalation_via_suid_binaries.toml + mitre: + - T1548.001 + - T1574.002 + 5984a354-d76c-43e6-bdd9-228456f1b371: + name: Persistence via Message-of-the-Day + path: ./linux/queries/persistence_via_message_of_the_day.toml + mitre: + - T1036.005 + - T1546.003 + 00461198-9a2d-4823-b4cc-f3d1b5c17935: + name: Hidden Process Execution + path: ./linux/queries/defense_evasion_via_hidden_process_execution.toml + mitre: + - T1036.004 + - T1059 + 6e57e6a6-f150-405d-b8be-e4e666a3a86d: + name: Privilege Escalation Identification via Existing Sudoers File + path: ./linux/queries/privilege_escalation_via_existing_sudoers.toml + mitre: + - T1548.003 + 223f812c-a962-4d58-961d-134d8f8b15da: + name: Excessive SSH Network Activity to Unique Destinations + path: ./linux/queries/excessive_ssh_network_activity_unique_destinations.toml + mitre: + - T1021.004 + - T1078.003 + 8dcc2161-65e0-4448-a03a-1c4e0cbc9330: + name: XDG Persistence + path: ./linux/queries/persistence_via_xdg_autostart_modifications.toml + mitre: + - T1547.001 + - T1053.005 + d2d24ad6-a315-4e05-a3f9-e205eb805df4: + name: Persistence via Systemd (Timers) + path: ./linux/queries/persistence_via_systemd_timers.toml + mitre: + - T1053.005 + - T1546.002 + 12526f14-5e35-4f5f-884c-96c6a353a544: + name: Low Volume External Network Connections from Process by Unique Agent + path: ./linux/queries/low_volume_external_network_connections_from_process.toml + mitre: + - T1071.001 + - T1071.004 + 27d76f07-7dc4-49bc-b4a7-6d9a01de171f: + name: Persistence via System V Init + path: ./linux/queries/persistence_via_sysv_init.toml + mitre: + - T1037 + 2d7bb29d-d53f-47ab-a0b4-1818adb91423: + name: Git Hook/Pager Persistence + path: ./linux/queries/persistence_via_git_hook_pager.toml + mitre: + - T1546.004 + - T1059.004 + 7422faf1-ba51-49c3-b8ba-13759e6bcec4: + name: Persistence Through Reverse/Bind Shells + path: ./linux/queries/persistence_reverse_bind_shells.toml + mitre: + - T1059.004 + c7044817-d9a5-4755-abab-9059e50dab24: + name: Low Volume Modifications to Critical System Binaries by Unique Host + path: ./linux/queries/low_volume_modifications_to_critical_system_binaries.toml + mitre: + - T1070.004 + - T1569.002 + 20a02fad-2a09-44c0-a8ce-ce4502859c8a: + name: Shell Modification Persistence + path: ./linux/queries/persistence_via_shell_modification_persistence.toml + mitre: + - T1546.004 + - T1053.005 + 0ea47044-b161-4785-ba99-e11f46d6ac51: + name: Uncommon Process Execution from Suspicious Directory + path: ./linux/queries/execution_uncommon_process_execution_from_suspicious_directory.toml + mitre: + - T1036.004 + - T1049 + - T1059 + - T1059.004 + 783d6091-b98d-45a8-a880-a07f112a8aa2: + name: Low Volume GTFOBins External Network Connections + path: ./linux/queries/low_volume_gtfobins_external_network_connections.toml + mitre: + - T1219 + - T1071.001 + 8d42a644-5b60-4165-a8f1-84d5bcdd4ade: + name: Persistence via Udev + path: ./linux/queries/persistence_via_udev.toml + mitre: + - T1547.010 + e1f59c9a-7a2a-4eb8-a524-97b16a041a4a: + name: Drivers Load with Low Occurrence Frequency + path: ./linux/queries/persistence_via_driver_load_with_low_occurrence_frequency.toml + mitre: + - T1547.006 + - T1069.002 + 95c1467d-d566-4645-b5f1-37a4b0093bb6: + name: Logon Activity by Source IP + path: ./linux/queries/login_activity_by_source_address.toml + mitre: + - T1110 + - T1078 + d22cbe8f-c84d-4811-aa6d-f1ee00c806b2: + name: Unusual System Binary Parent (Potential System Binary Hijacking Attempt) + path: ./linux/queries/persistence_via_unusual_system_binary_parent.toml + mitre: + - T1546.004 + - T1059.004 + 3f3fd2b9-940c-4310-adb1-d8b7d726e281: + name: Segmentation Fault & Potential Buffer Overflow Hunting + path: ./linux/queries/privilege_escalation_via_segmentation_fault_and_buffer_overflow.toml + mitre: + - T1203 + - T1068 + 2d01a413-8d97-407a-8698-02dfc7119c97: + name: Persistence via Package Manager + path: ./linux/queries/persistence_via_package_manager.toml + mitre: + - T1546.004 + - T1059.004 + 11810497-8ce3-4960-9777-9d0e97052682: + name: Potential Defense Evasion via Multi-Dot Process Execution + path: ./linux/queries/defense_evasion_via_multi_dot_process_execution.toml + mitre: + - T1036.004 + - T1070 + 0d061fad-cf35-43a6-b9b7-986c348bf182: + name: Unusual File Downloads from Source Addresses + path: ./linux/queries/command_and_control_via_unusual_file_downloads_from_source_addresses.toml + mitre: + - T1071.001 + - T1071.004 + 6f67704d-e5b1-4613-912c-e2965660fe17: + name: Process Capability Hunting + path: ./linux/queries/privilege_escalation_via_process_capabilities.toml + mitre: + - T1548.001 + - T1548.003 + aa759db0-4499-42f2-9f2f-be3e00fdebfa: + name: Persistence via SSH Configurations and/or Keys + path: ./linux/queries/persistence_via_ssh_configurations_and_keys.toml + mitre: + - T1098.004 + - T1563.001 + e1cffb7c-4acf-4e7a-8d72-b8b7657cf7b8: + name: Persistence via Cron + path: ./linux/queries/persistence_via_cron.toml + mitre: + - T1053.003 + - T1053.005 + c9931736-d5ec-4c89-b4d2-d71dcf5ca12a: + name: Low Volume Process Injection-Related Syscalls by Process Executable + path: ./linux/queries/low_volume_process_injection_syscalls_by_executable.toml + mitre: + - T1055.001 + - T1055.009 + f00c9757-d21b-432c-90a6-8372f18075d0: + name: Privilege Escalation/Persistence via User/Group Creation and/or Modification + path: ./linux/queries/persistence_via_user_group_creation_modification.toml + mitre: + - T1136 + - T1136.001 + - T1136.002 + 9d485892-1ca2-464b-9e4e-6b21ab379b9a: + name: Defense Evasion via Capitalized Process Execution + path: ./linux/queries/defense_evasion_via_capitalized_process_execution.toml + mitre: + - T1036.004 + - T1070 + a95f778f-2193-4a3d-bbbe-7b02d5740638: + name: Persistence via rc.local/rc.common + path: ./linux/queries/persistence_via_rc_local.toml + mitre: + - T1037.004 + - T1546.003 +okta: + 0b936024-71d9-11ef-a9be-f661ea17fbcc: + name: Failed OAuth Access Token Retrieval via Public Client App + path: ./okta/queries/defense_evasion_failed_oauth_access_token_retrieval_via_public_client_app.toml + mitre: + - T1550.001 + 31585786-71f4-11ef-9e99-f661ea17fbcc: + name: Successful Impossible Travel Sign-On Events + path: ./okta/queries/initial_access_impossible_travel_sign_on.toml + mitre: + - T1078.004 + 223451b0-6eca-11ef-a070-f661ea17fbcc: + name: Rapid MFA Deny Push Notifications (MFA Bombing) + path: ./okta/queries/credential_access_mfa_bombing_push_notications.toml + mitre: + - T1621 + 11666aa0-71d9-11ef-a9be-f661ea17fbcc: + name: Rare Occurrence of OAuth Access Token Granted to Public Client App + path: ./okta/queries/defense_evasion_rare_oauth_access_token_granted_by_application.toml + mitre: + - T1550.001 + c8a35a26-71f1-11ef-9c4e-f661ea17fbcc: + name: Identify High Average of Failed Daily Authentication Attempts + path: ./okta/queries/initial_access_higher_than_average_failed_authentication.toml + mitre: + - T1078.004 + 1c2d2b08-71ee-11ef-952e-f661ea17fbcc: + name: Password Spraying from Repeat Source + path: ./okta/queries/initial_access_password_spraying_from_repeat_source.toml + mitre: + - T1078.004 + f3bc68f4-71e9-11ef-952e-f661ea17fbcc: + name: Rare Occurrence of Domain with User Authentication Events + path: ./okta/queries/persistence_rare_domain_with_user_authentication.toml + mitre: + - T1078.004 + 7c51fe3e-6ae9-11ef-919d-f661ea17fbcc: + name: Multi-Factor Authentication (MFA) Push Notification Bombing + path: ./okta/queries/persistence_multi_factor_push_notification_bombing.toml + mitre: + - T1556.006 + c784106e-6ae8-11ef-919d-f661ea17fbcc: + name: Rapid Reset Password Requests for Different Users + path: ./okta/queries/credential_access_rapid_reset_password_requests_for_different_users.toml + mitre: + - T1098.001 + 38d82c2c-71d9-11ef-a9be-f661ea17fbcc: + name: OAuth Access Token Granted for Public Client App from Multiple Client Addresses + path: ./okta/queries/defense_evasion_multiple_client_sources_reported_for_oauth_access_tokens_granted.toml + mitre: + - T1550.001 + 03bce3b0-6ded-11ef-9282-f661ea17fbcc: + name: Multiple Application SSO Authentication from the Same Source + path: ./okta/queries/defense_evasion_multiple_application_sso_authentication_repeat_source.toml + mitre: + - T1550.001 +aws: + c3d24ae8-655d-11ef-a990-f661ea17fbcc: + name: High EC2 Instance Deployment Count Attempts by Single User or Role + path: ./aws/queries/ec2_high_instance_deployment_count_attempts.toml + mitre: + - T1578.002 + e3206d1c-64a9-11ef-a642-f661ea17fbcc: + name: Lambda Add Permissions for Write Actions to Function + path: ./aws/queries/lambda_add_permissions_for_write_actions_to_function.toml + mitre: + - T1584.007 + 913a47be-649c-11ef-a693-f661ea17fbcc: + name: IAM User Activity with No MFA Session + path: ./aws/queries/iam_user_activity_with_no_mfa_session.toml + mitre: + - T1078.004 + f9eae44e-5e4d-11ef-878f-f661ea17fbce: + name: SSM Start Remote Session to EC2 Instance + path: ./aws/queries/ssm_start_remote_session_to_ec2_instance.toml + mitre: + - T1021.007 + e6e78858-6482-11ef-93bd-f661ea17fbcc: + name: High Frequency of EC2 Multi-Region `DescribeInstances` API Calls + path: ./aws/queries/ec2_discovery_multi_region_describe_instance_calls.toml + mitre: + - T1580 + 429824b6-60b2-11ef-b0a4-f661ea17fbce: + name: IAM Assume Role Creation with Attached Policy + path: ./aws/queries/iam_assume_role_creation_with_attached_policy.toml + mitre: + - T1098.003 + 1844f2d6-5dc7-11ef-b76c-f661ea17fbce: + name: SSM Rare SendCommand Code Execution by EC2 Instance + path: ./aws/queries/ssm_rare_sendcommand_code_execution.toml + mitre: + - T1651 + f11ac62c-5f42-11ef-9d72-f661ea17fbce: + name: EC2 Modify Instance Attribute User Data + path: ./aws/queries/ec2_modify_instance_attribute_user_data.toml + mitre: + - T1059.009 + - T1037 + ef579900-75ef-11ef-b47f-f661ea17fbcc: + name: S3 Public Bucket Rapid Object Access Attempts + path: ./aws/queries/s3_public_bucket_rapid_object_access_attempts.toml + mitre: + - T1530 + 408ba5f6-5db7-11ef-a01c-f661ea17fbce: + name: EC2 Suspicious Get User Password Request + path: ./aws/queries/ec2_suspicious_get_user_password_request.toml + mitre: + - T1552.005 + 38454a64-5b55-11ef-b345-f661ea17fbce: + name: SSM SendCommand API Used by EC2 Instance + path: ./aws/queries/ssm_sendcommand_api_used_by_ec2_instance.toml + mitre: + - T1651 + 953b1252-5efd-11ef-a997-f661ea17fbce: + name: Signin Single Factor Console Login via Federated Session + path: ./aws/queries/signin_single_factor_console_login_via_federated_session.toml + mitre: + - T1078.004 + d74f8928-5e46-11ef-9488-f661ea17fbce: + name: Multiple Service Logging Deleted or Stopped + path: ./aws/queries/multiple_service_logging_deleted_or_stopped.toml + mitre: + - T1562.008 + ef244ca0-5e32-11ef-a8d3-f661ea17fbce: + name: Secrets Manager High Frequency of Programmatic GetSecretValue API Calls + path: ./aws/queries/secretsmanager_high_frequency_get_secret_value.toml + mitre: + - T1555.006 + 7a083b24-6482-11ef-8a8f-f661ea17fbcc: + name: High Frequency of Service Quotas Multi-Region `GetServiceQuota` API Calls + path: ./aws/queries/servicequotas_discovery_multi_region_get_service_quota_calls.toml + mitre: + - T1580 + 696c3f40-5b54-11ef-b9df-f661ea17fbce: + name: User Creation with Administrator Policy Assigned + path: ./aws/queries/iam_user_creation_with_administrator_policy_assigned.toml + mitre: + - T1098.003 + - T1136.003 + 3f8393b2-5f0b-11ef-8a25-f661ea17fbce: + name: STS Suspicious Federated Temporary Credential Request + path: ./aws/queries/sts_suspicious_federated_temporary_credential_request.toml + mitre: + - T1550.001 +windows: + 44e6adc6-e183-4bfa-b06d-db41669641fa: + name: Rundll32 Execution Aggregated by Command Line + path: ./windows/queries/rundll32_execution_aggregated_by_cmdline.toml + mitre: + - T1127 + - T1218 + - T1218.011 + df4ee961-254d-4ad1-af15-c65c3b65abcd: + name: Persistence via Run Key with Low Occurrence Frequency + path: ./windows/queries/persistence_via_run_key_with_low_occurrence_frequency.toml + mitre: + - T1547 + - T1547.001 + 5e5aa9c2-96a8-4d5b-bbca-ff2ec8fefa5b: + name: High Count of Network Connection Over Extended Period by Process + path: ./windows/queries/high_count_of_network_connection_over_extended_period_by_process.toml + mitre: + - T1071 + 4f878255-53b8-4914-9a7d-4b668bd2ea6a: + name: Low Occurrence Rate of CreateRemoteThread by Source Process + path: ./windows/queries/createremotethread_by_source_process_with_low_occurrence.toml + mitre: + - T1055 + 34a7aadb-fb0f-45ea-9260-830f39c3343b: + name: Rare DLL Side-Loading by Occurrence + path: ./windows/queries/detect_rare_dll_sideload_by_occurrence.toml + mitre: + - T1574 + - T1574.002 + f7d2054f-b571-4cd0-b39e-a779576e9398: + name: Excessive RDP Network Activity by Host and User + path: ./windows/queries/excessive_rdp_network_activity_by_source_host_and_user.toml + mitre: + - T1021 + - T1021.001 + d06bc067-6174-412f-b1c9-bf8f15149519: + name: DLL Hijack via Masquerading as Microsoft Native Libraries + path: ./windows/queries/detect_dll_hijack_via_masquerading_as_microsoft_native_libraries.toml + mitre: + - T1574 + - T1574.001 + 44223fd6-8241-4c21-9d54-21201fa15b12: + name: Scheduled Tasks Creation for Unique Hosts by Task Command + path: ./windows/queries/scheduled_tasks_creation_for_unique_hosts_by_task_command.toml + mitre: + - T1053 + - T1053.005 + 24925575-defd-4581-bfda-a8753dcfb46e: + name: Egress Network Connections with Total Bytes Greater than Threshold + path: ./windows/queries/potential_exfiltration_by_process_total_egress_bytes.toml + mitre: + - T1071 + df50f65e-e820-47f4-a039-671611582f51: + name: Scheduled tasks Creation by Action via Registry + path: ./windows/queries/scheduled_task_creation_by_action_via_registry.toml + mitre: + - T1053 + - T1053.005 + a95e69af-22ad-4ab7-919e-794501f10c95: + name: Low Frequency of Process Execution via WMI by Unique Agent + path: ./windows/queries/execution_via_windows_management_instrumentation_by_occurrence_frequency_by_unique_agent.toml + mitre: + - T1047 + 1c7be6db-12eb-4281-878d-b6abe0454f36: + name: DNS Queries via LOLBins with Low Occurence Frequency + path: ./windows/queries/domain_names_queried_via_lolbins_and_with_low_occurence_frequency.toml + mitre: + - T1071 + 386f9cec-bb44-4dd2-8368-45e6fa0a425b: + name: Network Discovery via Sensitive Ports by Unusual Process + path: ./windows/queries/network_discovery_via_sensitive_ports_by_unusual_process.toml + mitre: + - T1021 + - T1021.002 + - T1021.001 + 48b75e53-3c73-40bd-873d-569dd8d7d925: + name: Unique Windows Services Creation by Service File Name + path: ./windows/queries/unique_windows_services_creation_by_servicefilename.toml + mitre: + - T1543 + - T1543.003 + 7a2c8397-d219-47ad-a8e2-93562e568d08: + name: Suspicious DNS TXT Record Lookups by Process + path: ./windows/queries/suspicious_dns_txt_record_lookups_by_process.toml + mitre: + - T1071 + - T1071.004 + ea950361-33e4-4045-96a5-d36ca28fbc91: + name: Persistence via Startup with Low Occurrence Frequency by Unique Host + path: ./windows/queries/persistence_via_startup_with_low_occurrence_frequency.toml + mitre: + - T1547 + - T1547.001 + d0aed6f5-f84c-4da8-bb2a-b5ca0fbb55e0: + name: Rare LSASS Process Access Attempts + path: ./windows/queries/detect_rare_lsass_process_access_attempts.toml + mitre: + - T1003 + - T1003.001 + 24108755-4d1f-4d7a-ad5f-04c2ca55e9a3: + name: Frequency of Process Execution via Network Logon by Source Address + path: ./windows/queries/execution_via_network_logon_by_occurrence_frequency_by_top_source_ip.toml + mitre: + - T1021 + c00f1afe-4f25-4542-8cc9-277b23581121: + name: Libraries Loaded by svchost with Low Occurrence Frequency + path: ./windows/queries/libraries_loaded_by_svchost_with_low_occurrence_frequency.toml + mitre: + - T1543 + - T1543.003 + a0a84a86-115f-42f9-90a5-4cb7ceeef981: + name: Low Occurence of Process Execution via Windows Services with Unique Agent + path: ./windows/queries/execution_via_windows_services_with_low_occurrence_frequency.toml + mitre: + - T1543 + - T1543.003 + 52a958e8-0368-4e74-bd4b-a64faf397bf4: + name: Startup Execution with Low Occurrence Frequency by Unique Host + path: ./windows/queries/execution_via_startup_with_low_occurrence_frequency.toml + mitre: + - T1547 + - T1547.001 + a2006c66-d6ab-43ee-871e-d650e38f7972: + name: Masquerading Attempts as Native Windows Binaries + path: ./windows/queries/detect_masquerading_attempts_as_native_windows_binaries.toml + mitre: + - T1036 + 2e583d3c-7ad6-4544-a0db-c685b2066493: + name: Suspicious Base64 Encoded Powershell Command + path: ./windows/queries/suspicious_base64_encoded_powershell_commands.toml + mitre: + - T1059 + - T1059.001 + - T1027 + - T1027.010 + cebfbb4d-5b2a-44d8-b763-5512b654fb26: + name: Low Occurrence of Drivers Loaded on Unique Hosts + path: ./windows/queries/drivers_load_with_low_occurrence_frequency.toml + mitre: + - T1068 + 441fba85-47a9-4f1f-aab4-569bbfdc548b: + name: Windows Logon Activity by Source IP + path: ./windows/queries/windows_logon_activity_by_source_ip.toml + mitre: + - T1110 + - T1110.001 + - T1110.003 + b786bcd7-b119-4ff7-b839-3927c2ff7f1f: + name: Executable File Creation by an Unusual Microsoft Binary + path: ./windows/queries/executable_file_creation_by_an_unusual_microsoft_binary.toml + mitre: + - T1211 + - T1055 + 0d960760-8a40-49c1-bbdd-4deb32c7fd67: + name: Low Frequency of Process Execution via Windows Scheduled Task by Unique + Agent + path: ./windows/queries/execution_via_windows_scheduled_task_with_low_occurrence_frequency.toml + mitre: + - T1053 + - T1053.005 + 5fd5da54-0515-4d6b-b8d7-30fd05f5be33: + name: Execution via Remote Services by Client Address + path: ./windows/queries/execution_via_remote_services_by_client_address.toml + mitre: + - T1021 + - T1021.003 + - T1021.006 + - T1047 + aca4877f-d284-4bdb-8e18-b1414d3a7c20: + name: Windows Command and Scripting Interpreter from Unusual Parent Process + path: ./windows/queries/windows_command_and_scripting_interpreter_from_unusual_parent.toml + mitre: + - T1059 + - T1059.001 + - T1059.003 + 814894a4-c951-4f33-ab0b-09354e1cb957: + name: PE File Transfer via SMB_Admin Shares by Agent or User + path: ./windows/queries/pe_file_transfer_via_smb_admin_shares_by_agent.toml + mitre: + - T1021 + - T1021.002 + f1b8519a-4dae-475f-965a-f53559233eab: + name: Microsoft Office Child Processes with Low Occurrence Frequency by Unique + Agent + path: ./windows/queries/microsoft_office_child_processes_with_low_occurrence_frequency.toml + mitre: + - T1566 + - T1566.001 + 8a95f552-f149-4c71-888e-f2690f5add15: + name: Excessive SMB Network Activity by Process ID + path: ./windows/queries/excessive_smb_network_activity_by_process_id.toml + mitre: + - T1021 + - T1021.002 diff --git a/hunting/markdown.py b/hunting/markdown.py new file mode 100644 index 000000000..9a139e8f8 --- /dev/null +++ b/hunting/markdown.py @@ -0,0 +1,144 @@ +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0; you may not use this file except in compliance with the Elastic License +# 2.0. + +from pathlib import Path +import click +from .definitions import ATLAS_URL, ATTACK_URL, STATIC_INTEGRATION_LINK_MAP, Hunt +from .utils import load_index_file, load_toml, save_index_file, validate_link + + +class MarkdownGenerator: + """Class to generate or update Markdown documentation from TOML or YAML files.""" + def __init__(self, base_path: Path): + """Initialize with the base path and load the hunting index.""" + self.base_path = base_path + self.hunting_index = load_index_file() + + def process_file(self, file_path: Path) -> None: + """Process a single TOML file and generate its Markdown representation.""" + if not file_path.is_file() or file_path.suffix != '.toml': + raise ValueError(f"The provided path is not a valid TOML file: {file_path}") + + click.echo(f"Processing specific TOML file: {file_path}") + hunt_config = load_toml(file_path) + markdown_content = self.convert_toml_to_markdown(hunt_config, file_path) + + docs_folder = self.create_docs_folder(file_path) + markdown_path = docs_folder / f"{file_path.stem}.md" + self.save_markdown(markdown_path, markdown_content) + + self.update_or_add_entry(hunt_config, file_path) + + def process_folder(self, folder: str) -> None: + """Process all TOML files in a specified folder and generate their Markdown representations.""" + folder_path = self.base_path / folder / "queries" + docs_folder = self.base_path / folder / "docs" + + if not folder_path.is_dir() or not docs_folder.is_dir(): + raise ValueError(f"Queries folder {folder_path} or docs folder {docs_folder} does not exist.") + + click.echo(f"Processing all TOML files in folder: {folder_path}") + toml_files = folder_path.rglob("*.toml") + + for toml_file in toml_files: + self.process_file(toml_file) + + def process_all_files(self) -> None: + """Process all TOML files in the base directory and subfolders.""" + click.echo("Processing all TOML files in the base directory and subfolders.") + toml_files = self.base_path.rglob("queries/*.toml") + + for toml_file in toml_files: + self.process_file(toml_file) + + def convert_toml_to_markdown(self, hunt_config: Hunt, file_path: Path) -> str: + """Convert a Hunt configuration to Markdown format.""" + markdown = f"# {hunt_config.name}\n\n---\n\n" + markdown += "## Metadata\n\n" + markdown += f"- **Author:** {hunt_config.author}\n" + markdown += f"- **Description:** {hunt_config.description}\n" + markdown += f"- **UUID:** `{hunt_config.uuid}`\n" + markdown += f"- **Integration:** {', '.join(self.generate_integration_links(hunt_config.integration))}\n" + markdown += f"- **Language:** `{hunt_config.language}`\n".replace("'", "").replace('"', "") + markdown += f"- **Source File:** [{hunt_config.name}]({(Path('../queries') / file_path.name).as_posix()})\n" + markdown += "\n## Query\n\n" + for query in hunt_config.query: + markdown += f"```sql\n{query}```\n\n" + + if hunt_config.notes: + markdown += "## Notes\n\n" + "\n".join(f"- {note}" for note in hunt_config.notes) + if hunt_config.mitre: + markdown += "\n\n## MITRE ATT&CK Techniques\n\n" + "\n".join( + f"- [{tech}]({ATLAS_URL if tech.startswith('AML') else ATTACK_URL}" + f"{tech.replace('.', '/') if tech.startswith('T') else tech})" + for tech in hunt_config.mitre + ) + if hunt_config.references: + markdown += "\n\n## References\n\n" + "\n".join(f"- {ref}" for ref in hunt_config.references) + + markdown += f"\n\n## License\n\n- `{hunt_config.license}`\n" + return markdown + + def save_markdown(self, markdown_path: Path, content: str) -> None: + """Save the Markdown content to a file.""" + markdown_path.write_text(content, encoding="utf-8") + click.echo(f"Markdown generated: {markdown_path}") + + def update_or_add_entry(self, hunt_config: Hunt, toml_path: Path) -> None: + """Update or add the entry for a TOML file in the hunting index.""" + folder_name = toml_path.parent.parent.name + uuid = hunt_config.uuid + + entry = { + 'name': hunt_config.name, + 'path': f"./{toml_path.relative_to(self.base_path).as_posix()}", + 'mitre': hunt_config.mitre + } + + if folder_name not in self.hunting_index: + self.hunting_index[folder_name] = {uuid: entry} + else: + self.hunting_index[folder_name][uuid] = entry + + save_index_file(self.base_path, self.hunting_index) + + def create_docs_folder(self, file_path: Path) -> Path: + """Create the docs folder if it doesn't exist and return the path.""" + docs_folder = file_path.parent.parent / "docs" + docs_folder.mkdir(parents=True, exist_ok=True) + return docs_folder + + def generate_integration_links(self, integrations: list[str]) -> list[str]: + """Generate integration links for the documentation.""" + base_url = 'https://docs.elastic.co/integrations' + generated = [] + for integration in integrations: + if integration in STATIC_INTEGRATION_LINK_MAP: + link_str = STATIC_INTEGRATION_LINK_MAP[integration] + else: + link_str = integration.replace('.', '/') + link = f'{base_url}/{link_str}' + validate_link(link) + generated.append(f'[{integration}]({link})') + return generated + + def update_index_md(self) -> None: + """Update the index.md file based on the entries in index.yml.""" + index_file = self.base_path / "index.yml" + index_content = "# List of Available Queries\n\nHere are the queries currently available:\n" + + if not index_file.exists(): + click.echo(f"No index.yml found at {index_file}. Skipping index.md update.") + return + + for folder, files in sorted(self.hunting_index.items()): + index_content += f"\n\n## {folder}\n" + for file_info in sorted(files.values(), key=lambda x: x['name']): + md_path = file_info['path'].replace('queries', 'docs').replace('.toml', '.md') + index_content += f"- [{file_info['name']}]({md_path}) (ES|QL)\n" + + index_md_path = self.base_path / "index.md" + index_md_path.write_text(index_content, encoding="utf-8") + click.echo(f"Index Markdown updated at: {index_md_path}") diff --git a/hunting/okta/docs/docs/credential_access_mfa_bombing_push_notications.md b/hunting/okta/docs/credential_access_mfa_bombing_push_notications.md similarity index 100% rename from hunting/okta/docs/docs/credential_access_mfa_bombing_push_notications.md rename to hunting/okta/docs/credential_access_mfa_bombing_push_notications.md diff --git a/hunting/okta/docs/docs/credential_access_rapid_reset_password_requests_for_different_users.md b/hunting/okta/docs/credential_access_rapid_reset_password_requests_for_different_users.md similarity index 100% rename from hunting/okta/docs/docs/credential_access_rapid_reset_password_requests_for_different_users.md rename to hunting/okta/docs/credential_access_rapid_reset_password_requests_for_different_users.md diff --git a/hunting/okta/docs/docs/defense_evasion_failed_oauth_access_token_retrieval_via_public_client_app.md b/hunting/okta/docs/defense_evasion_failed_oauth_access_token_retrieval_via_public_client_app.md similarity index 100% rename from hunting/okta/docs/docs/defense_evasion_failed_oauth_access_token_retrieval_via_public_client_app.md rename to hunting/okta/docs/defense_evasion_failed_oauth_access_token_retrieval_via_public_client_app.md diff --git a/hunting/okta/docs/docs/defense_evasion_multiple_application_sso_authentication_repeat_source.md b/hunting/okta/docs/defense_evasion_multiple_application_sso_authentication_repeat_source.md similarity index 100% rename from hunting/okta/docs/docs/defense_evasion_multiple_application_sso_authentication_repeat_source.md rename to hunting/okta/docs/defense_evasion_multiple_application_sso_authentication_repeat_source.md diff --git a/hunting/okta/docs/docs/defense_evasion_multiple_client_sources_reported_for_oauth_access_tokens_granted.md b/hunting/okta/docs/defense_evasion_multiple_client_sources_reported_for_oauth_access_tokens_granted.md similarity index 100% rename from hunting/okta/docs/docs/defense_evasion_multiple_client_sources_reported_for_oauth_access_tokens_granted.md rename to hunting/okta/docs/defense_evasion_multiple_client_sources_reported_for_oauth_access_tokens_granted.md diff --git a/hunting/okta/docs/docs/defense_evasion_rare_oauth_access_token_granted_by_application.md b/hunting/okta/docs/defense_evasion_rare_oauth_access_token_granted_by_application.md similarity index 100% rename from hunting/okta/docs/docs/defense_evasion_rare_oauth_access_token_granted_by_application.md rename to hunting/okta/docs/defense_evasion_rare_oauth_access_token_granted_by_application.md diff --git a/hunting/okta/docs/docs/credential_access_hgher_than_average_failed_authentication.md b/hunting/okta/docs/docs/credential_access_hgher_than_average_failed_authentication.md deleted file mode 100644 index 64d277d83..000000000 --- a/hunting/okta/docs/docs/credential_access_hgher_than_average_failed_authentication.md +++ /dev/null @@ -1,58 +0,0 @@ -# Identify High Average of Failed Daily Authentication Attempts - ---- - -## Metadata - -- **Author:** Elastic -- **Description:** This hunting query identifies when the average number of failed daily authentication attempts is higher than normal in Okta. Adversaries may attempt to brute force user credentials to gain unauthorized access to accounts. This query calculates the average number of daily failed authentication attempts for each user and identifies when the average is higher than normal. - -- **UUID:** `c8a35a26-71f1-11ef-9c4e-f661ea17fbcc` -- **Integration:** [okta](https://docs.elastic.co/integrations/okta) -- **Language:** `[ES|QL]` -- **Source File:** [Identify High Average of Failed Daily Authentication Attempts](../queries/credential_access_hgher_than_average_failed_authentication.toml) - -## Query - -```sql -from logs-okta* -| where @timestamp > NOW() - 7 day - -// truncate the timestamp to daily intervals -| eval target_time_window = DATE_TRUNC(1 days, @timestamp) -| where - - // filter for invalid credential authentication events - event.action == "user.session.start" - and okta.outcome.result == "FAILURE" - and okta.outcome.reason == "INVALID_CREDENTIALS" - -// add a count of 1 for each failed login -| eval failed_login_count = 1 - -| stats - // count the number of daily failed logins for each day and user - failed_daily_logins = count(*) by target_time_window, okta.actor.alternate_id - -| stats - // calculate the average number of daily failed logins for each day - avg_daily_logins = avg(failed_daily_logins) by target_time_window - -// sort the results by the average number of daily failed logins in descending order -| sort avg_daily_logins desc -``` - -## Notes - -- Pivot to users by only keeping the first stats statement where `okta.actor.alternate_id` is the targeted accounts. -- Pivot for successful logins from the same source IP by searching for `event.action` equal to `user.session.start` or `user.authentication.verify` where the outcome is `SUCCESS`. -- User agents can be used to identify anomalous behavior, such as a user agent that is not associated with a known application or user. -- Another `WHERE` count can be added to the query if activity has been baseline to filter out known behavior. - -## MITRE ATT&CK Techniques - -- [T1078.004](https://attack.mitre.org/techniques/T1078/004) - -## License - -- `Elastic License v2` diff --git a/hunting/okta/docs/docs/credential_access_password_spraying_from_repeat_source.md b/hunting/okta/docs/docs/credential_access_password_spraying_from_repeat_source.md deleted file mode 100644 index 1a43784c5..000000000 --- a/hunting/okta/docs/docs/credential_access_password_spraying_from_repeat_source.md +++ /dev/null @@ -1,53 +0,0 @@ -# Password Spraying from Repeat Source - ---- - -## Metadata - -- **Author:** Elastic -- **Description:** This hunting query identifies password spraying attacks in Okta where the same source IP attempts to authenticate to multiple accounts with invalid credentials. Adversaries may attempt to use a single source IP to avoid detection and bypass account lockout policies. - -- **UUID:** `1c2d2b08-71ee-11ef-952e-f661ea17fbcc` -- **Integration:** [okta](https://docs.elastic.co/integrations/okta) -- **Language:** `[ES|QL]` -- **Source File:** [Password Spraying from Repeat Source](../queries/credential_access_password_spraying_from_repeat_source.toml) - -## Query - -```sql -from logs-okta* -| where @timestamp > NOW() - 7 day - -// truncate the timestamp to daily intervals -| eval target_time_window = DATE_TRUNC(1 days, @timestamp) -| where - -// filter for invalid credential events - event.action == "user.session.start" - and okta.outcome.result == "FAILURE" - and okta.outcome.reason == "INVALID_CREDENTIALS" -| stats - // count the distinct number of targeted accounts - target_count = count_distinct(okta.actor.alternate_id), - - // count the number of invalid credential events for each source IP - source_count = count(*) by target_time_window, okta.client.ip - -// filter for source IPs with more than 5 invalid credential events -| where target_count >= 5 -``` - -## Notes - -- `okta.actor.alternate_id` are the targeted accounts. -- Pivot for successful logins from the same source IP by searching for `event.action` equal to `user.session.start` or `user.authentication.verify` where the outcome is `SUCCESS`. -- User agents can be used to identify anomalous behavior, such as a user agent that is not associated with a known application or user. -- The `target_count` can be adjusted depending on the organization's account lockout policy or baselined behavior. - -## MITRE ATT&CK Techniques - -- [T1078.004](https://attack.mitre.org/techniques/T1078/004) - -## License - -- `Elastic License v2` diff --git a/hunting/okta/docs/docs/initial_access_hgher_than_average_failed_authentication.md b/hunting/okta/docs/docs/initial_access_hgher_than_average_failed_authentication.md deleted file mode 100644 index bf30579e2..000000000 --- a/hunting/okta/docs/docs/initial_access_hgher_than_average_failed_authentication.md +++ /dev/null @@ -1,59 +0,0 @@ -# Identify High Average of Failed Daily Authentication Attempts - ---- - -## Metadata - -- **Author:** Elastic -- **Description:** This hunting query identifies when the average number of failed daily authentication attempts is higher than normal in Okta. Adversaries may attempt to brute force user credentials to gain unauthorized access to accounts. This query calculates the average number of daily failed authentication attempts for each user and identifies when the average is higher than normal. - -- **UUID:** `c8a35a26-71f1-11ef-9c4e-f661ea17fbcc` -- **Integration:** [okta](https://docs.elastic.co/integrations/okta) -- **Language:** `[ES|QL]` -- **Source File:** [Identify High Average of Failed Daily Authentication Attempts](../queries/initial_access_hgher_than_average_failed_authentication.toml) - -## Query - -```sql -from logs-okta* -| where @timestamp > NOW() - 7 day - -// truncate the timestamp to daily intervals -| eval target_time_window = DATE_TRUNC(1 days, @timestamp) -| where - - // filter for invalid credential authentication events - event.action == "user.session.start" - and okta.outcome.result == "FAILURE" - and okta.outcome.reason == "INVALID_CREDENTIALS" - and okta.actor.type == "User" - -// add a count of 1 for each failed login -| eval failed_login_count = 1 - -| stats - // count the number of daily failed logins for each day and user - failed_daily_logins = count(*) by target_time_window, okta.actor.alternate_id - -| stats - // calculate the average number of daily failed logins for each day - avg_daily_logins = avg(failed_daily_logins) by target_time_window - -// sort the results by the average number of daily failed logins in descending order -| sort avg_daily_logins desc -``` - -## Notes - -- Pivot to users by only keeping the first stats statement where `okta.actor.alternate_id` is the targeted accounts. -- Pivot for successful logins from the same source IP by searching for `event.action` equal to `user.session.start` or `user.authentication.verify` where the outcome is `SUCCESS`. -- User agents can be used to identify anomalous behavior, such as a user agent that is not associated with a known application or user. -- Another `WHERE` count can be added to the query if activity has been baseline to filter out known behavior. - -## MITRE ATT&CK Techniques - -- [T1078.004](https://attack.mitre.org/techniques/T1078/004) - -## License - -- `Elastic License v2` diff --git a/hunting/okta/docs/docs/persistence_rare_tld_with_user_authentication.md b/hunting/okta/docs/docs/persistence_rare_tld_with_user_authentication.md deleted file mode 100644 index 220b5d138..000000000 --- a/hunting/okta/docs/docs/persistence_rare_tld_with_user_authentication.md +++ /dev/null @@ -1,48 +0,0 @@ -# Rare Occurrence of Top-Level Domain (TLD) with User Authentication Events - ---- - -## Metadata - -- **Author:** Elastic -- **Description:** This hunting query identifies when a top-level domain (TLD) has a rare occurrence of user authentication events in Okta. Adversaries may leverage compromised Okta accounts or tokens with admin privileges to create new users that are registered with an adversary-controlled email address. - -- **UUID:** `f3bc68f4-71e9-11ef-952e-f661ea17fbcc` -- **Integration:** [okta](https://docs.elastic.co/integrations/okta) -- **Language:** `[ES|QL]` -- **Source File:** [Rare Occurrence of Top-Level Domain (TLD) with User Authentication Events](../queries/persistence_rare_tld_with_user_authentication.toml) - -## Query - -```sql -from logs-okta* -| where @timestamp > NOW() - 7 day -| where - // Filter for user authentication events - okta.actor.alternate_id is not null - and event.action LIKE "user.authentication*" - -// Extract the top-level domain (TLD) from the user's email address -| dissect okta.actor.alternate_id "%{}@%{tld}" - -// Count the number of user authentication events for each TLD -| stats tld_auth_counts = count(*) by tld - -// Filter for TLDs with less than or equal to 5 user authentication events -| where tld_auth_counts <= 5 - -// Sort the results by the number of user authentication events in ascending order -| sort tld_auth_counts asc -``` - -## Notes - -- Pivot into potential compromised accounts by searching for the `okta.actor.alternate_id` in `okta.target` where `event.action` is `user.lifecycle.create`. This would identify when the user account was created. The `okta.actor.alternate_id` of this event will also be the potential compromised account. - -## MITRE ATT&CK Techniques - -- [T1078.004](https://attack.mitre.org/techniques/T1078/004) - -## License - -- `Elastic License v2` diff --git a/hunting/okta/docs/docs/initial_access_higher_than_average_failed_authentication.md b/hunting/okta/docs/initial_access_higher_than_average_failed_authentication.md similarity index 100% rename from hunting/okta/docs/docs/initial_access_higher_than_average_failed_authentication.md rename to hunting/okta/docs/initial_access_higher_than_average_failed_authentication.md diff --git a/hunting/okta/docs/docs/initial_access_impossible_travel_sign_on.md b/hunting/okta/docs/initial_access_impossible_travel_sign_on.md similarity index 100% rename from hunting/okta/docs/docs/initial_access_impossible_travel_sign_on.md rename to hunting/okta/docs/initial_access_impossible_travel_sign_on.md diff --git a/hunting/okta/docs/docs/initial_access_password_spraying_from_repeat_source.md b/hunting/okta/docs/initial_access_password_spraying_from_repeat_source.md similarity index 100% rename from hunting/okta/docs/docs/initial_access_password_spraying_from_repeat_source.md rename to hunting/okta/docs/initial_access_password_spraying_from_repeat_source.md diff --git a/hunting/okta/docs/docs/persistence_multi_factor_push_notification_bombing.md b/hunting/okta/docs/persistence_multi_factor_push_notification_bombing.md similarity index 100% rename from hunting/okta/docs/docs/persistence_multi_factor_push_notification_bombing.md rename to hunting/okta/docs/persistence_multi_factor_push_notification_bombing.md diff --git a/hunting/okta/docs/docs/persistence_rare_domain_with_user_authentication.md b/hunting/okta/docs/persistence_rare_domain_with_user_authentication.md similarity index 100% rename from hunting/okta/docs/docs/persistence_rare_domain_with_user_authentication.md rename to hunting/okta/docs/persistence_rare_domain_with_user_authentication.md diff --git a/hunting/okta/docs/queries/credential_access_mfa_bombing_push_notications.toml b/hunting/okta/queries/credential_access_mfa_bombing_push_notications.toml similarity index 100% rename from hunting/okta/docs/queries/credential_access_mfa_bombing_push_notications.toml rename to hunting/okta/queries/credential_access_mfa_bombing_push_notications.toml diff --git a/hunting/okta/docs/queries/credential_access_rapid_reset_password_requests_for_different_users.toml b/hunting/okta/queries/credential_access_rapid_reset_password_requests_for_different_users.toml similarity index 100% rename from hunting/okta/docs/queries/credential_access_rapid_reset_password_requests_for_different_users.toml rename to hunting/okta/queries/credential_access_rapid_reset_password_requests_for_different_users.toml diff --git a/hunting/okta/docs/queries/defense_evasion_failed_oauth_access_token_retrieval_via_public_client_app.toml b/hunting/okta/queries/defense_evasion_failed_oauth_access_token_retrieval_via_public_client_app.toml similarity index 100% rename from hunting/okta/docs/queries/defense_evasion_failed_oauth_access_token_retrieval_via_public_client_app.toml rename to hunting/okta/queries/defense_evasion_failed_oauth_access_token_retrieval_via_public_client_app.toml diff --git a/hunting/okta/docs/queries/defense_evasion_multiple_application_sso_authentication_repeat_source.toml b/hunting/okta/queries/defense_evasion_multiple_application_sso_authentication_repeat_source.toml similarity index 100% rename from hunting/okta/docs/queries/defense_evasion_multiple_application_sso_authentication_repeat_source.toml rename to hunting/okta/queries/defense_evasion_multiple_application_sso_authentication_repeat_source.toml diff --git a/hunting/okta/docs/queries/defense_evasion_multiple_client_sources_reported_for_oauth_access_tokens_granted.toml b/hunting/okta/queries/defense_evasion_multiple_client_sources_reported_for_oauth_access_tokens_granted.toml similarity index 100% rename from hunting/okta/docs/queries/defense_evasion_multiple_client_sources_reported_for_oauth_access_tokens_granted.toml rename to hunting/okta/queries/defense_evasion_multiple_client_sources_reported_for_oauth_access_tokens_granted.toml diff --git a/hunting/okta/docs/queries/defense_evasion_rare_oauth_access_token_granted_by_application.toml b/hunting/okta/queries/defense_evasion_rare_oauth_access_token_granted_by_application.toml similarity index 100% rename from hunting/okta/docs/queries/defense_evasion_rare_oauth_access_token_granted_by_application.toml rename to hunting/okta/queries/defense_evasion_rare_oauth_access_token_granted_by_application.toml diff --git a/hunting/okta/docs/queries/initial_access_higher_than_average_failed_authentication.toml b/hunting/okta/queries/initial_access_higher_than_average_failed_authentication.toml similarity index 100% rename from hunting/okta/docs/queries/initial_access_higher_than_average_failed_authentication.toml rename to hunting/okta/queries/initial_access_higher_than_average_failed_authentication.toml diff --git a/hunting/okta/docs/queries/initial_access_impossible_travel_sign_on.toml b/hunting/okta/queries/initial_access_impossible_travel_sign_on.toml similarity index 100% rename from hunting/okta/docs/queries/initial_access_impossible_travel_sign_on.toml rename to hunting/okta/queries/initial_access_impossible_travel_sign_on.toml diff --git a/hunting/okta/docs/queries/initial_access_password_spraying_from_repeat_source.toml b/hunting/okta/queries/initial_access_password_spraying_from_repeat_source.toml similarity index 100% rename from hunting/okta/docs/queries/initial_access_password_spraying_from_repeat_source.toml rename to hunting/okta/queries/initial_access_password_spraying_from_repeat_source.toml diff --git a/hunting/okta/docs/queries/persistence_multi_factor_push_notification_bombing.toml b/hunting/okta/queries/persistence_multi_factor_push_notification_bombing.toml similarity index 100% rename from hunting/okta/docs/queries/persistence_multi_factor_push_notification_bombing.toml rename to hunting/okta/queries/persistence_multi_factor_push_notification_bombing.toml diff --git a/hunting/okta/docs/queries/persistence_rare_domain_with_user_authentication.toml b/hunting/okta/queries/persistence_rare_domain_with_user_authentication.toml similarity index 100% rename from hunting/okta/docs/queries/persistence_rare_domain_with_user_authentication.toml rename to hunting/okta/queries/persistence_rare_domain_with_user_authentication.toml diff --git a/hunting/run.py b/hunting/run.py new file mode 100644 index 000000000..17ce29914 --- /dev/null +++ b/hunting/run.py @@ -0,0 +1,76 @@ +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0; you may not use this file except in compliance with the Elastic License +# 2.0. + +import re +import textwrap +from pathlib import Path + +import click + +from detection_rules.misc import get_elasticsearch_client + +from .utils import load_toml + + +class QueryRunner: + def __init__(self, es_config: dict): + """Initialize the QueryRunner with Elasticsearch config.""" + self.es_config = es_config + + def load_hunting_file(self, file_path: Path): + """Load the hunting file and return the data.""" + return load_toml(file_path) + + def preprocess_query(self, query: str) -> str: + """Pre-process the query by removing comments and adding a LIMIT.""" + query = re.sub(r'//.*', '', query) + if not re.search(r'LIMIT', query, re.IGNORECASE): + query += " | LIMIT 10" + click.echo("No LIMIT detected in query. Added LIMIT 10 to truncate output.") + return query + + def run_individual_query(self, query: str, wait_timeout: int): + """Run a single query with the Elasticsearch config.""" + es = get_elasticsearch_client(**self.es_config) + query = self.preprocess_query(query) + + try: + click.secho("Running query. Press Ctrl+C to cancel.", fg="blue") + query = query.replace("\n", " ") + + # Start the query synchronously + response = es.esql.query(query=query) + self.process_results(response) + except Exception as e: + # handle missing index error + if "Unknown index" in str(e): + click.secho("This query references indexes that do not exist in the target stack.", fg="red") + click.secho("Check if index exists (via integration installation) and contains data.", fg="red") + click.secho("Alternatively, update the query to reference an existing index.", fg="red") + else: + click.secho(f"Error running query: {str(e)}", fg="red") + + def run_all_queries(self, queries: dict, wait_timeout: int): + """Run all eligible queries in the hunting file.""" + click.secho("Running all eligible queries...", fg="green", bold=True) + for i, query in queries.items(): + click.secho(f"\nRunning Query {i + 1}:", fg="green", bold=True) + click.echo(self._format_query(query)) + self.run_individual_query(query, wait_timeout) + click.secho("\n" + "-" * 120, fg="yellow") + + def process_results(self, response): + """Process the Elasticsearch query results and display the outcome.""" + response_data = response.body + if response_data.get('values'): + click.secho("Query matches found!", fg="red", bold=True) + else: + click.secho("No matches found!", fg="green", bold=True) + + def _format_query(self, query: str) -> str: + """Format the query with word wrapping for better readability.""" + lines = query.split('\n') + wrapped_lines = [textwrap.fill(line, width=120, subsequent_indent=' ') for line in lines] + return '\n'.join(wrapped_lines) diff --git a/hunting/search.py b/hunting/search.py new file mode 100644 index 000000000..615e8cd22 --- /dev/null +++ b/hunting/search.py @@ -0,0 +1,188 @@ +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0; you may not use this file except in compliance with the Elastic License +# 2.0. + + +from pathlib import Path +import click +from detection_rules.attack import tactics_map, technique_lookup +from .utils import load_index_file, load_all_toml + + +class QueryIndex: + def __init__(self, base_path: Path): + """Initialize with the base path and load the index.""" + self.base_path = base_path + self.hunting_index = load_index_file() + self.mitre_technique_ids = set() + self.reverse_tactics_map = {v: k for k, v in tactics_map.items()} + + def _process_mitre_filter(self, mitre_filter: tuple): + """Process the MITRE filter to gather all matching techniques.""" + for filter_item in mitre_filter: + if filter_item in self.reverse_tactics_map: + self._process_tactic_id(filter_item) + elif filter_item in technique_lookup: + self._process_technique_id(filter_item) + + def _process_tactic_id(self, filter_item): + """Helper method to process a tactic ID.""" + tactic_name = self.reverse_tactics_map[filter_item] + click.echo(f"Found tactic ID {filter_item} (Tactic Name: {tactic_name}). Searching for associated techniques.") + + for tech_id, details in technique_lookup.items(): + kill_chain_phases = details.get('kill_chain_phases', []) + if any(tactic_name.lower().replace(' ', '-') == phase['phase_name'] for phase in kill_chain_phases): + self.mitre_technique_ids.add(tech_id) + + def _process_technique_id(self, filter_item): + """Helper method to process a technique or sub-technique ID.""" + self.mitre_technique_ids.add(filter_item) + if '.' not in filter_item: + sub_techniques = { + sub_tech_id for sub_tech_id in technique_lookup + if sub_tech_id.startswith(f"{filter_item}.") + } + self.mitre_technique_ids.update(sub_techniques) + + def search(self, mitre_filter: tuple = (), data_source: str = None, keyword: str = None) -> list: + """Search the index based on MITRE techniques, data source, or keyword.""" + results = [] + + # Step 1: If data source is provided, filter by data source first + if data_source: + click.echo(f"Filtering by data source: {data_source}") + results = self._filter_by_data_source(data_source) + if not results: + # data source always takes precedence over other filters if provided + click.echo(f"No matching queries found for data source: {data_source}") + return results + + # Step 2: If MITRE filter is provided, process the filter + if mitre_filter: + click.echo(f"Searching for MITRE techniques: {mitre_filter}") + self._process_mitre_filter(mitre_filter) + if results: + # Filter existing results further by MITRE if data source results already exist + results = [result for result in results if + any(tech in self.mitre_technique_ids for tech in result['mitre'])] + else: + # Otherwise, perform a fresh search based on MITRE filter + results = self._search_index(mitre_filter) + + # Step 3: If keyword is provided, search for it in name, description, and notes + if keyword: + click.echo(f"Searching for keyword: {keyword}") + if results: + # Filter existing results further by keyword + results = [result for result in results if self._matches_keyword(result, keyword)] + else: + # Perform a fresh search by keyword + results = self._search_keyword(keyword) + + return self._handle_no_results(results, mitre_filter, data_source, keyword) + + def _search_index(self, mitre_filter: tuple = ()) -> list: + """Private method to search the index based on MITRE filter.""" + results = [] + # Load all TOML data for detailed fields + hunting_content = load_all_toml(self.base_path) + + for hunt_content, file_path in hunting_content: + query_techniques = hunt_content.mitre + if mitre_filter and not any(tech in self.mitre_technique_ids for tech in query_techniques): + continue + + # Prepare the result with full hunt content fields + matches = hunt_content.__dict__.copy() + matches['mitre'] = hunt_content.mitre + matches['data_source'] = hunt_content.integration + matches['uuid'] = hunt_content.uuid + matches['path'] = file_path + results.append(matches) + + return results + + def _search_keyword(self, keyword: str) -> list: + """Private method to search description, name, notes, and references fields for a keyword.""" + results = [] + hunting_content = load_all_toml(self.base_path) + + for hunt_content, file_path in hunting_content: + # Assign blank if notes or references are missing + notes = '::'.join(hunt_content.notes) if hunt_content.notes else '' + references = '::'.join(hunt_content.references) if hunt_content.references else '' + + # Combine name, description, notes, and references for the search + combined_content = f"{hunt_content.name}::{hunt_content.description}::{notes}::{references}" + + if keyword.lower() in combined_content.lower(): + # Copy hunt_content data and prepare the result + matches = hunt_content.__dict__.copy() + matches['mitre'] = hunt_content.mitre + matches['data_source'] = hunt_content.integration + matches['uuid'] = hunt_content.uuid + matches['path'] = file_path + results.append(matches) + + return results + + def _filter_by_data_source(self, data_source: str) -> list: + """Filter the index by data source, checking both the actual files and the index.""" + results = [] + seen_uuids = set() # Track UUIDs to avoid duplicates + + # Load all TOML data for detailed fields + hunting_content = load_all_toml(self.base_path) + + # Step 1: Check files first by their 'integration' field + for hunt_content, file_path in hunting_content: + if data_source in hunt_content.integration: + if hunt_content.uuid not in seen_uuids: + # Prepare the result with full hunt content fields + matches = hunt_content.__dict__.copy() + matches['mitre'] = hunt_content.mitre + matches['data_source'] = hunt_content.integration + matches['uuid'] = hunt_content.uuid + matches['path'] = file_path + results.append(matches) + seen_uuids.add(hunt_content.uuid) + + # Step 2: Check the index for generic data sources (e.g., 'aws', 'linux') + if data_source in self.hunting_index: + for query_uuid, query_data in self.hunting_index[data_source].items(): + if query_uuid not in seen_uuids: + # Find corresponding TOML content for this query + hunt_content = next((hunt for hunt, path in hunting_content if hunt.uuid == query_uuid), None) + if hunt_content: + # Prepare the result with full hunt content fields + matches = hunt_content.__dict__.copy() + matches['mitre'] = hunt_content.mitre + matches['data_source'] = hunt_content.integration + matches['uuid'] = hunt_content.uuid + matches['path'] = file_path + results.append(matches) + seen_uuids.add(query_uuid) + + return results + + def _matches_keyword(self, result: dict, keyword: str) -> bool: + """Check if the result matches the keyword in name, description, or notes.""" + # Combine relevant fields for keyword search + notes = '::'.join(result.get('notes', [])) if 'notes' in result else '' + references = '::'.join(result.get('references', [])) if 'references' in result else '' + combined_content = f"{result['name']}::{result['description']}::{notes}::{references}" + + return keyword.lower() in combined_content.lower() + + def _handle_no_results(self, results: list, mitre_filter=None, data_source=None, keyword=None) -> list: + """Handle cases where no results are found.""" + if not results: + if mitre_filter and not self.mitre_technique_ids: + click.echo(f"No MITRE techniques found for the provided filter: {mitre_filter}.") + if data_source: + click.echo(f"No matching queries found for data source: {data_source}") + if keyword: + click.echo(f"No matches found for keyword: {keyword}") + return results diff --git a/hunting/utils.py b/hunting/utils.py new file mode 100644 index 000000000..c704d1624 --- /dev/null +++ b/hunting/utils.py @@ -0,0 +1,134 @@ +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0; you may not use this file except in compliance with the Elastic License +# 2.0. + +import inspect +import tomllib +from pathlib import Path +from typing import Union + +import click +import urllib3 +import yaml + +from detection_rules.misc import get_elasticsearch_client + +from .definitions import HUNTING_DIR, Hunt + + +def get_hunt_path(uuid: str, file_path: str) -> (Path, str): + """Resolve the path of the hunting query using either a UUID or file path.""" + + if uuid: + # Load the index and find the hunt by UUID + index_data = load_index_file() + for data_source, hunts in index_data.items(): + if uuid in hunts: + hunt_data = hunts[uuid] + # Combine the relative path from the index with the HUNTING_DIR + hunt_path = HUNTING_DIR / hunt_data['path'] + return hunt_path.resolve(), None + return None, f"No hunt found for UUID: {uuid}" + + elif file_path: + # Use the provided file path + hunt_path = Path(file_path) + if not hunt_path.is_file(): + return None, f"No file found at path: {file_path}" + return hunt_path.resolve(), None + + return None, "Either UUID or file path must be provided." + + +def load_index_file() -> dict: + """Load the hunting index.yml file.""" + index_file = HUNTING_DIR / "index.yml" + if not index_file.exists(): + click.echo(f"No index.yml found at {index_file}.") + return {} + + with open(index_file, 'r') as f: + hunting_index = yaml.safe_load(f) + + return hunting_index + + +def load_toml(source: Union[Path, str]) -> Hunt: + """Load and validate TOML content as Hunt dataclass.""" + if isinstance(source, Path): + if not source.is_file(): + raise FileNotFoundError(f"TOML file not found: {source}") + contents = source.read_text(encoding="utf-8") + else: + contents = source + + toml_dict = tomllib.loads(contents) + + # Validate and load the content into the Hunt dataclass + return Hunt(**toml_dict["hunt"]) + + +def load_all_toml(base_path: Path): + """Load all TOML files from the directory and return a list of Hunt configurations and their paths.""" + hunts = [] + for toml_file in base_path.rglob("*.toml"): + hunt_config = load_toml(toml_file) + hunts.append((hunt_config, toml_file)) + return hunts + + +def save_index_file(base_path: Path, directories: dict) -> None: + """Save the updated index.yml file.""" + index_file = base_path / "index.yml" + with open(index_file, 'w') as f: + yaml.safe_dump(directories, f, default_flow_style=False, sort_keys=False) + print(f"Index YAML updated at: {index_file}") + + +def validate_link(link: str): + """Validate and return the link.""" + http = urllib3.PoolManager() + response = http.request('GET', link) + if response.status != 200: + raise ValueError(f"Invalid link: {link}") + + +def update_index_yml(base_path: Path) -> None: + """Update index.yml based on the current TOML files.""" + directories = load_index_file() + + # Load all TOML files recursively + toml_files = base_path.rglob("queries/*.toml") + + for toml_file in toml_files: + # Load TOML and extract hunt configuration + hunt_config = load_toml(toml_file) + + folder_name = toml_file.parent.parent.name + uuid = hunt_config.uuid + + entry = { + 'name': hunt_config.name, + 'path': f"./{toml_file.relative_to(base_path).as_posix()}", + 'mitre': hunt_config.mitre + } + + # Check if the folder_name exists and if it's a list, convert it to a dictionary + if folder_name not in directories: + directories[folder_name] = {uuid: entry} + else: + if isinstance(directories[folder_name], list): + # Convert the list to a dictionary, using UUIDs as keys + directories[folder_name] = {item['uuid']: item for item in directories[folder_name]} + directories[folder_name][uuid] = entry + + # Save the updated index.yml + save_index_file(base_path, directories) + + +def filter_elasticsearch_params(config: dict) -> dict: + """Filter out unwanted keys from the config by inspecting the Elasticsearch client constructor.""" + # Get the parameter names from the Elasticsearch class constructor + es_params = inspect.signature(get_elasticsearch_client).parameters + return {k: v for k, v in config.items() if k in es_params} diff --git a/pyproject.toml b/pyproject.toml index 27e5ad3f3..aa1ebcc73 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,6 +43,7 @@ dependencies = [ ] [project.optional-dependencies] dev = ["pep8-naming==0.13.0", "PyGithub==2.2.0", "flake8==7.0.0", "pyflakes==3.2.0", "pytest>=8.1.1", "nodeenv==1.8.0", "pre-commit==3.6.2"] +hunting = ["tabulate==0.9.0"] [project.urls] "Homepage" = "https://github.com/elastic/detection-rules" @@ -52,7 +53,7 @@ dev = ["pep8-naming==0.13.0", "PyGithub==2.2.0", "flake8==7.0.0", "pyflakes==3.2 [tool.setuptools] package-data = {"kql" = ["*.g"]} -packages = ["detection_rules", "rta"] +packages = ["detection_rules", "rta", "hunting"] [tool.pytest.ini_options] filterwarnings = [ diff --git a/tests/test_hunt_data.py b/tests/test_hunt_data.py index 2c39fccb4..d4a6b3e9e 100644 --- a/tests/test_hunt_data.py +++ b/tests/test_hunt_data.py @@ -6,7 +6,9 @@ """Test for hunt toml files.""" import unittest -from hunting.generate_markdown import HUNTING_DIR, load_toml +from hunting.definitions import HUNTING_DIR +from hunting.markdown import load_toml +from hunting.utils import load_index_file, load_all_toml class TestHunt(unittest.TestCase): @@ -31,7 +33,6 @@ class TestHunt(unittest.TestCase): config = load_toml(example_toml) self.assertEqual(config.author, "Elastic") self.assertEqual(config.integration, "aws_bedrock.invocation") - self.assertEqual(config.uuid, "dc181967-c32c-46c9-b84b-ec4c8811c6a0") self.assertEqual( config.name, "Denial of Service or Resource Exhaustion Attacks Detection" ) @@ -40,13 +41,11 @@ class TestHunt(unittest.TestCase): def test_load_toml_files(self): """Test loading and validating all Hunt TOML files in the hunting directory.""" - for toml_file in HUNTING_DIR.rglob("*.toml"): - toml_contents = toml_file.read_text() - hunt = load_toml(toml_contents) + for toml_path in HUNTING_DIR.rglob("*.toml"): + hunt = load_toml(toml_path) self.assertTrue(hunt.author) self.assertTrue(hunt.description) self.assertTrue(hunt.integration) - self.assertTrue(hunt.uuid) self.assertTrue(hunt.name) self.assertTrue(hunt.language) self.assertTrue(hunt.query) @@ -63,6 +62,59 @@ class TestHunt(unittest.TestCase): f"Markdown file not found for {toml_file} at expected location {expected_markdown_path}", ) + def test_toml_existence(self): + """Ensure each Markdown file has a corresponding TOML file in the queries directory.""" + for markdown_file in HUNTING_DIR.rglob("*/docs/*.md"): + expected_toml_path = ( + markdown_file.parent.parent / "queries" / markdown_file.with_suffix(".toml").name + ) + + self.assertTrue( + expected_toml_path.exists(), + f"TOML file not found for {markdown_file} at expected location {expected_toml_path}", + ) + + +class TestHuntIndex(unittest.TestCase): + """Test the hunting index.yml file.""" + + @classmethod + def setUpClass(cls): + """Load the index once for all tests.""" + cls.hunting_index = load_index_file() + + def test_mitre_techniques_present(self): + """Ensure each query has at least one MITRE technique.""" + for folder, queries in self.hunting_index.items(): + for query_uuid, query_data in queries.items(): + self.assertTrue(query_data.get('mitre'), + f"No MITRE techniques found for query: {query_data.get('name', query_uuid)}") + + def test_valid_structure(self): + """Ensure each query entry has a valid structure.""" + required_fields = ['name', 'path', 'mitre'] + + for folder, queries in self.hunting_index.items(): + for query_uuid, query_data in queries.items(): + for field in required_fields: + self.assertIn(field, query_data, f"Missing field '{field}' in query: {query_data}") + + def test_all_files_in_index(self): + """Ensure all TOML files are included in the index.""" + missing_index_entries = [] + all_toml_data = load_all_toml(HUNTING_DIR) + uuids = [hunt.uuid for hunt, path in all_toml_data] + + for folder, queries in self.hunting_index.items(): + for query_uuid in queries: + if query_uuid not in uuids: + missing_index_entries.append(query_uuid) + + self.assertFalse( + missing_index_entries, + f"Missing index entries for the following queries: {missing_index_entries}" + ) + if __name__ == "__main__": unittest.main()