From 88e0b14709b0b2d583fcdf6fb7c68e6aae27df7a Mon Sep 17 00:00:00 2001 From: Samirbous <64742097+Samirbous@users.noreply.github.com> Date: Mon, 26 Jan 2026 16:34:16 +0000 Subject: [PATCH] [Tuning] ESQL Dynamic unique value fields (#5569) * [Tuning] Extract dynamic field with 1 value to ECS fields for alerts exclusion Extract dynamic field with 1 value to ECS fields for alerts exclusion: Esql.host_id_values -> host.is Esql.agent_id_values -> agent.id Esql.host_name_values -> host.name * Update multiple_alerts_by_host_ip_and_source_ip.toml * Update newly_observed_elastic_defend_alert.toml * Update defense_evasion_base64_decoding_activity.toml * Update discovery_subnet_scanning_activity_from_compromised_host.toml * Update persistence_web_server_sus_command_execution.toml * Update persistence_web_server_sus_child_spawned.toml * Update rules/cross-platform/multiple_alerts_elastic_defend_netsecurity_by_host.toml Co-authored-by: Mika Ayenson, PhD * Update rules/linux/impact_potential_bruteforce_malware_infection.toml Co-authored-by: Mika Ayenson, PhD * Update rules/linux/command_and_control_frequent_egress_netcon_from_sus_executable.toml Co-authored-by: Mika Ayenson, PhD * Apply suggestions from code review Co-authored-by: Mika Ayenson, PhD * Update rules/cross-platform/multiple_alerts_elastic_defend_netsecurity_by_host.toml Co-authored-by: Mika Ayenson, PhD * Update rules/cross-platform/newly_observed_elastic_defend_alert.toml Co-authored-by: Mika Ayenson, PhD * Update rules/cross-platform/newly_observed_elastic_detection_rule.toml Co-authored-by: Mika Ayenson, PhD * Update rules/windows/credential_access_rare_webdav_destination.toml Co-authored-by: Mika Ayenson, PhD * Update credential_access_rare_webdav_destination.toml --------- Co-authored-by: Mika Ayenson, PhD --- .../multiple_alerts_by_host_ip_and_source_ip.toml | 8 ++++++-- ...le_alerts_elastic_defend_netsecurity_by_host.toml | 10 ++++++++-- .../newly_observed_elastic_defend_alert.toml | 10 +++++++--- .../newly_observed_elastic_detection_rule.toml | 8 ++++++-- ...l_frequent_egress_netcon_from_sus_executable.toml | 9 ++++++++- .../defense_evasion_base64_decoding_activity.toml | 8 +++++++- ...port_scanning_activity_from_compromised_host.toml | 8 +++++++- ...bnet_scanning_activity_from_compromised_host.toml | 8 +++++++- ...ation_unusual_file_transfer_utility_launched.toml | 8 +++++++- ...mpact_potential_bruteforce_malware_infection.toml | 9 ++++++++- .../persistence_web_server_sus_child_spawned.toml | 8 +++++++- ...persistence_web_server_sus_command_execution.toml | 8 +++++++- .../credential_access_lsass_openprocess_api.toml | 12 +++++++++--- 13 files changed, 94 insertions(+), 20 deletions(-) diff --git a/rules/cross-platform/multiple_alerts_by_host_ip_and_source_ip.toml b/rules/cross-platform/multiple_alerts_by_host_ip_and_source_ip.toml index 2249581b6..eab228720 100644 --- a/rules/cross-platform/multiple_alerts_by_host_ip_and_source_ip.toml +++ b/rules/cross-platform/multiple_alerts_by_host_ip_and_source_ip.toml @@ -1,7 +1,7 @@ [metadata] creation_date = "2025/12/31" maturity = "production" -updated_date = "2025/12/31" +updated_date = "2026/01/16" [rule] author = ["Elastic"] @@ -60,7 +60,11 @@ from .alerts-security.* | eval concat_ip_values = MV_CONCAT(TO_STRING(Esql.host_ip_values), ",") | eval host_ip_equal_to_source_ip =LOCATE(concat_ip_values, TO_STRING(Esql.source_ip)) | where Esql.rule_name_distinct_count >= 2 and Esql.host_id_distinct_count >= 2 and host_ip_equal_to_source_ip > 0 and SRC_IP is not null and Esql.alerts_count <= 100 -| KEEP Esql.* + +// Move single values to their corresponding ECS fields for alerts exclusion +| eval source.ip = mv_min(Esql.source_ip), + host.id = mv_min(Esql.host_id_values) +| KEEP Esql.*, source.ip, host.id ''' note = """## Triage and analysis diff --git a/rules/cross-platform/multiple_alerts_elastic_defend_netsecurity_by_host.toml b/rules/cross-platform/multiple_alerts_elastic_defend_netsecurity_by_host.toml index 899d601fa..e55dc0902 100644 --- a/rules/cross-platform/multiple_alerts_elastic_defend_netsecurity_by_host.toml +++ b/rules/cross-platform/multiple_alerts_elastic_defend_netsecurity_by_host.toml @@ -2,7 +2,7 @@ creation_date = "2025/11/18" integration = ["endpoint", "panw", "fortinet_fortigate", "suricata"] maturity = "production" -updated_date = "2025/12/30" +updated_date = "2026/01/16" [rule] author = ["Elastic"] @@ -73,7 +73,13 @@ FROM logs-* metadata _id | eval concat_module_values = MV_CONCAT(Esql.event_module_values, ",") // Make sure an endpoint alert is present along one of the network ones | where concat_module_values like "*endpoint*" -| keep Esql.* + +// Move single values to their corresponding ECS fields for alerts exclusion +| eval source.ip = mv_min(Esql.source_ip), + host.id = mv_min(Esql.host_id_values), + user.name = mv_min(Esql.user_name_values) + +| keep source.ip, host.id, user.name, Esql.* ''' note = """## Triage and analysis diff --git a/rules/cross-platform/newly_observed_elastic_defend_alert.toml b/rules/cross-platform/newly_observed_elastic_defend_alert.toml index e1782a108..d367d17f7 100644 --- a/rules/cross-platform/newly_observed_elastic_defend_alert.toml +++ b/rules/cross-platform/newly_observed_elastic_defend_alert.toml @@ -1,7 +1,7 @@ [metadata] creation_date = "2026/01/05" maturity = "production" -updated_date = "2026/01/05" +updated_date = "2026/01/16" [rule] author = ["Elastic"] @@ -33,13 +33,17 @@ from logs-endpoint.alerts-* Esql.process_parent_executable = VALUES(process.parent.executable), Esql.process_command_line = VALUES(process.command_line), Esql.process_hash_sha256 = VALUES(process.hash.sha256), - Esql.host_id = VALUES(host.id), + Esql.host_id_values = VALUES(host.id), Esql.user_name = VALUES(user.name) by rule.name // first time seen in the last 5 days - defined in the rule schedule Additional look-back time | eval Esql.recent = DATE_DIFF("minute", Esql.first_time_seen, now()) // first time seen is within 10m of the rule execution time | where Esql.recent <= 10 and Esql.agents_distinct_count == 1 and Esql.alerts_count <= 10 and (Esql.last_time_seen == Esql.first_time_seen) -| keep rule.name, Esql.* + +// Move single values to their corresponding ECS fields for alerts exclusion +| eval host.id = mv_min(Esql.host_id_values) + +| keep host.id, rule.name, Esql.* ''' note = """## Triage and analysis diff --git a/rules/cross-platform/newly_observed_elastic_detection_rule.toml b/rules/cross-platform/newly_observed_elastic_detection_rule.toml index 9d2195c77..aa2420aed 100644 --- a/rules/cross-platform/newly_observed_elastic_detection_rule.toml +++ b/rules/cross-platform/newly_observed_elastic_detection_rule.toml @@ -1,7 +1,7 @@ [metadata] creation_date = "2026/01/07" maturity = "production" -updated_date = "2026/01/07" +updated_date = "2026/01/16" [rule] author = ["Elastic"] @@ -49,7 +49,11 @@ FROM .alerts-security.* | eval Esql.recent = DATE_DIFF("minute", Esql.first_time_seen, now()) // first time seen is within 10m of the rule execution time | where Esql.recent <= 10 and Esql.agents_distinct_count == 1 and Esql.alerts_count <= 10 and (Esql.last_time_seen == Esql.first_time_seen) -| keep kibana.alert.rule.name, Esql.* + +// Move single values to their corresponding ECS fields for alerts exclusion +| eval host.id = mv_min(Esql.host_id_values) + +| keep host.id, kibana.alert.rule.name, Esql.* ''' note = """## Triage and analysis diff --git a/rules/linux/command_and_control_frequent_egress_netcon_from_sus_executable.toml b/rules/linux/command_and_control_frequent_egress_netcon_from_sus_executable.toml index ffafa0153..64968e762 100644 --- a/rules/linux/command_and_control_frequent_egress_netcon_from_sus_executable.toml +++ b/rules/linux/command_and_control_frequent_egress_netcon_from_sus_executable.toml @@ -2,7 +2,7 @@ creation_date = "2025/02/20" integration = ["endpoint"] maturity = "production" -updated_date = "2025/12/17" +updated_date = "2026/01/16" [rule] author = ["Elastic"] @@ -152,7 +152,14 @@ from logs-endpoint.events.network-* metadata _id, _index, _version | where Esql.agent_id_count_distinct == 1 and Esql.event_count > 15 + +// Extract unique values to ECS fields for alerts exclusion +| eval agent.id = mv_min(Esql.agent_id_values), + host.name = mv_min(Esql.host_name_values) + | sort Esql.event_count asc + +| keep agent.id, host.name, process.executable, Esql.* ''' [[rule.threat]] diff --git a/rules/linux/defense_evasion_base64_decoding_activity.toml b/rules/linux/defense_evasion_base64_decoding_activity.toml index 75d2d93c9..375a8b62e 100644 --- a/rules/linux/defense_evasion_base64_decoding_activity.toml +++ b/rules/linux/defense_evasion_base64_decoding_activity.toml @@ -2,7 +2,7 @@ creation_date = "2025/02/21" integration = ["endpoint"] maturity = "production" -updated_date = "2025/12/17" +updated_date = "2026/01/16" [rule] author = ["Elastic"] @@ -162,6 +162,12 @@ from logs-endpoint.events.process-* metadata _id, _index, _version Esql.agent_id_count_distinct == 1 and Esql.event_count < 15 | sort Esql.event_count asc + +// Extract unique values to ECS fields for alerts exclusion +| eval agent.id = mv_min(Esql.agent_id_values), + host.name = mv_min(Esql.host_name_values) + +| keep agent.id, host.name, process.name, process.command_line, Esql.* ''' [[rule.threat]] diff --git a/rules/linux/discovery_port_scanning_activity_from_compromised_host.toml b/rules/linux/discovery_port_scanning_activity_from_compromised_host.toml index df58b8810..123057b6a 100644 --- a/rules/linux/discovery_port_scanning_activity_from_compromised_host.toml +++ b/rules/linux/discovery_port_scanning_activity_from_compromised_host.toml @@ -2,7 +2,7 @@ creation_date = "2025/03/04" integration = ["endpoint"] maturity = "production" -updated_date = "2025/12/18" +updated_date = "2026/01/16" [rule] author = ["Elastic"] @@ -148,6 +148,12 @@ from logs-endpoint.events.network-* metadata _id, _index, _version Esql.agent_id_count_distinct == 1 and Esql.destination_port_count_distinct > 100 | sort Esql.event_count asc + +// Extract unique values to ECS fields for alerts exclusion +| eval agent.id = mv_min(Esql.agent_id_values), + host.name = mv_min(Esql.host_name_values) + +| keep agent.id, host.name, process.executable, destination.ip, Esql.* ''' [[rule.threat]] diff --git a/rules/linux/discovery_subnet_scanning_activity_from_compromised_host.toml b/rules/linux/discovery_subnet_scanning_activity_from_compromised_host.toml index 09fec8276..6f3aab1b3 100644 --- a/rules/linux/discovery_subnet_scanning_activity_from_compromised_host.toml +++ b/rules/linux/discovery_subnet_scanning_activity_from_compromised_host.toml @@ -2,7 +2,7 @@ creation_date = "2025/03/04" integration = ["endpoint"] maturity = "production" -updated_date = "2025/12/18" +updated_date = "2026/01/16" [rule] author = ["Elastic"] @@ -139,6 +139,12 @@ from logs-endpoint.events.network-* metadata _id, _index, _version Esql.agent_id_count_distinct == 1 and Esql.destination_ip_count_distinct > 250 | sort Esql.event_count asc + +// Extract unique values to ECS fields for alerts exclusion +| eval agent.id = mv_min(Esql.agent_id_values), + host.name = mv_min(Esql.host_name_values) + +| keep agent.id, host.name, process.executable, Esql.* ''' [[rule.threat]] diff --git a/rules/linux/exfiltration_unusual_file_transfer_utility_launched.toml b/rules/linux/exfiltration_unusual_file_transfer_utility_launched.toml index 75753e7fb..f6d821c47 100644 --- a/rules/linux/exfiltration_unusual_file_transfer_utility_launched.toml +++ b/rules/linux/exfiltration_unusual_file_transfer_utility_launched.toml @@ -2,7 +2,7 @@ creation_date = "2025/02/21" integration = ["endpoint"] maturity = "production" -updated_date = "2025/12/19" +updated_date = "2026/01/16" [rule] author = ["Elastic"] @@ -160,6 +160,12 @@ from logs-endpoint.events.process-* metadata _id, _index, _version Esql.agent_id_count_distinct == 1 and Esql.event_count < 5 | sort Esql.event_count asc + +// Extract unique values to ECS fields for alerts exclusion +| eval agent.id = mv_min(Esql.agent_id_values), + host.name = mv_min(Esql.host_name_values) + +| keep agent.id, host.name, process.executable, process.parent.executable, Esql.* ''' [[rule.threat]] diff --git a/rules/linux/impact_potential_bruteforce_malware_infection.toml b/rules/linux/impact_potential_bruteforce_malware_infection.toml index 425ab4585..93c5361d3 100644 --- a/rules/linux/impact_potential_bruteforce_malware_infection.toml +++ b/rules/linux/impact_potential_bruteforce_malware_infection.toml @@ -2,7 +2,7 @@ creation_date = "2025/02/20" integration = ["endpoint"] maturity = "production" -updated_date = "2025/12/19" +updated_date = "2026/01/16" [rule] author = ["Elastic"] @@ -152,7 +152,14 @@ from logs-endpoint.events.network-* metadata _id, _index, _version | where Esql.agent_id_count_distinct == 1 and Esql.event_count >= 100 + +// Extract unique values to ECS fields for alerts exclusion +| eval agent.id = mv_min(Esql.agent_id_values), + host.name = mv_min(Esql.host_name_values) + | sort Esql.event_count asc + +| keep Esql.*, agent.id, host.name, process.executable, destination.port ''' [[rule.threat]] diff --git a/rules/linux/persistence_web_server_sus_child_spawned.toml b/rules/linux/persistence_web_server_sus_child_spawned.toml index 4820eef8b..9bf48a74d 100644 --- a/rules/linux/persistence_web_server_sus_child_spawned.toml +++ b/rules/linux/persistence_web_server_sus_child_spawned.toml @@ -2,7 +2,7 @@ creation_date = "2025/03/04" integration = ["endpoint"] maturity = "production" -updated_date = "2025/12/23" +updated_date = "2026/01/16" [rule] author = ["Elastic"] @@ -182,6 +182,12 @@ from logs-endpoint.events.process-* metadata _id, _index, _version Esql.agent_id_count_distinct == 1 and Esql.event_count < 5 | sort Esql.event_count asc + +// Extract unique values to ECS fields for alerts exclusion +| eval agent.id = mv_min(Esql.agent_id_values), + host.name = mv_min(Esql.host_name_values) + +| keep agent.id, host.name, process.executable, process.working_directory, process.parent.executable, Esql.* ''' [[rule.threat]] diff --git a/rules/linux/persistence_web_server_sus_command_execution.toml b/rules/linux/persistence_web_server_sus_command_execution.toml index d4e562687..bd1fd7cbd 100644 --- a/rules/linux/persistence_web_server_sus_command_execution.toml +++ b/rules/linux/persistence_web_server_sus_command_execution.toml @@ -2,7 +2,7 @@ creation_date = "2025/03/04" integration = ["endpoint"] maturity = "production" -updated_date = "2025/12/23" +updated_date = "2026/01/16" [rule] author = ["Elastic"] @@ -167,6 +167,12 @@ from logs-endpoint.events.process-* metadata _id, _index, _version Esql.agent_id_count_distinct == 1 and Esql.event_count < 5 | sort Esql.event_count asc + +// Extract unique values to ECS fields for alerts exclusion +| eval agent.id = mv_min(Esql.agent_id_values), + host.name = mv_min(Esql.host_name_values) + +| keep agent.id, host.name, process.command_line, process.working_directory, process.parent.executable, Esql.* ''' [[rule.threat]] diff --git a/rules/windows/credential_access_lsass_openprocess_api.toml b/rules/windows/credential_access_lsass_openprocess_api.toml index 6e37f1813..db07c2e3c 100644 --- a/rules/windows/credential_access_lsass_openprocess_api.toml +++ b/rules/windows/credential_access_lsass_openprocess_api.toml @@ -2,7 +2,7 @@ creation_date = "2023/03/02" integration = ["endpoint", "m365_defender"] maturity = "production" -updated_date = "2025/12/11" +updated_date = "2026/01/16" [transform] [[transform.osquery]] @@ -142,9 +142,15 @@ from logs-endpoint.events.api-*, logs-m365_defender.event-* metadata _id, _versi Esql.data_stream_namespace.values = VALUES(data_stream.namespace), Esql.user_name_values = VALUES(user.name) by Esql.process_path -// Limit to rare instances +// Limit to rare instances limited to 1 unique host | where Esql.count_distinct_hosts == 1 and Esql.access_count <= 3 -| keep Esql.* + +// Extract the single host ID and process into their corresponding ECS fields for alerts exclusion +| eval host.id = mv_min(Esql.host_id_values), + process.executable = mv_min(Esql.process_path) + +// Add the new field to the keep statement +| keep Esql.*, host.id, process.executable '''