diff --git a/detection_rules/etc/integration-manifests.json.gz b/detection_rules/etc/integration-manifests.json.gz index 05d368565..dfd098edb 100644 Binary files a/detection_rules/etc/integration-manifests.json.gz and b/detection_rules/etc/integration-manifests.json.gz differ diff --git a/detection_rules/etc/integration-schemas.json.gz b/detection_rules/etc/integration-schemas.json.gz index f2af11e8d..539644532 100644 Binary files a/detection_rules/etc/integration-schemas.json.gz and b/detection_rules/etc/integration-schemas.json.gz differ diff --git a/detection_rules/rule.py b/detection_rules/rule.py index 0b6b6982b..7f7f85898 100644 --- a/detection_rules/rule.py +++ b/detection_rules/rule.py @@ -1018,6 +1018,9 @@ class TOMLRuleContents(BaseRuleContents, MarshmallowDataclassMixin): if package["integration"] not in policy_templates: del package["integration"] + # remove duplicate entries + package_integrations = list({json.dumps(d, sort_keys=True): + d for d in package_integrations}.values()) obj.setdefault("related_integrations", package_integrations) def _convert_add_required_fields(self, obj: dict) -> None: diff --git a/detection_rules/schemas/definitions.py b/detection_rules/schemas/definitions.py index 5fafb31ed..91e37957d 100644 --- a/detection_rules/schemas/definitions.py +++ b/detection_rules/schemas/definitions.py @@ -27,7 +27,7 @@ VERSION_PATTERN = f'^{_version}$' MINOR_SEMVER = r'^\d+\.\d+$' BRANCH_PATTERN = f'{VERSION_PATTERN}|^master$' -NON_DATASET_PACKAGES = ['apm', 'endpoint', 'system', 'windows', 'cloud_defend'] +NON_DATASET_PACKAGES = ['apm', 'endpoint', 'system', 'windows', 'cloud_defend', 'network_traffic'] INTERVAL_PATTERN = r'^\d+[mshd]$' TACTIC_URL = r'^https://attack.mitre.org/tactics/TA[0-9]+/$' TECHNIQUE_URL = r'^https://attack.mitre.org/techniques/T[0-9]+/$' diff --git a/rules/network/command_and_control_accepted_default_telnet_port_connection.toml b/rules/network/command_and_control_accepted_default_telnet_port_connection.toml index 3f9873c49..9e8c417b9 100644 --- a/rules/network/command_and_control_accepted_default_telnet_port_connection.toml +++ b/rules/network/command_and_control_accepted_default_telnet_port_connection.toml @@ -1,6 +1,6 @@ [metadata] creation_date = "2020/02/18" -integration = ["endpoint"] +integration = ["network_traffic"] maturity = "production" min_stack_comments = "New fields added: required_fields, related_integrations, setup" min_stack_version = "8.3.0" @@ -24,7 +24,7 @@ false_positives = [ """, ] from = "now-9m" -index = ["auditbeat-*", "filebeat-*", "packetbeat-*", "logs-endpoint.events.*"] +index = ["packetbeat-*", "logs-network_traffic.*"] language = "kuery" license = "Elastic License v2" name = "Accepted Default Telnet Port Connection" @@ -45,8 +45,7 @@ timestamp_override = "event.ingested" type = "query" query = ''' -event.category:(network or network_traffic) and destination.port:23 - and network.direction:(inbound or ingress or outbound or egress) +event.dataset: network_traffic.flow and event.type: connection and not event.action:( flow_dropped or denied or deny or flow_terminated or timeout or Reject or network_flow) diff --git a/rules/network/command_and_control_cobalt_strike_beacon.toml b/rules/network/command_and_control_cobalt_strike_beacon.toml index b263ebec5..26a4188a7 100644 --- a/rules/network/command_and_control_cobalt_strike_beacon.toml +++ b/rules/network/command_and_control_cobalt_strike_beacon.toml @@ -1,6 +1,6 @@ [metadata] creation_date = "2020/07/06" -integration = ["endpoint"] +integration = ["network_traffic"] maturity = "production" min_stack_comments = "New fields added: required_fields, related_integrations, setup" min_stack_version = "8.3.0" @@ -20,7 +20,7 @@ false_positives = [ """, ] from = "now-9m" -index = ["auditbeat-*", "filebeat-*", "packetbeat-*", "logs-endpoint.events.*"] +index = ["packetbeat-*", "logs-network_traffic.*"] language = "lucene" license = "Elastic License v2" name = "Cobalt Strike Command and Control Beacon" @@ -40,7 +40,7 @@ timestamp_override = "event.ingested" type = "query" query = ''' -event.category:(network OR network_traffic) AND type:(tls OR http) AND network.transport:tcp AND destination.domain:/[a-z]{3}.stage.[0-9]{8}\..*/ +event.dataset: (network_traffic.tls or network_traffic.http) AND destination.domain:/[a-z]{3}.stage.[0-9]{8}\..*/ ''' diff --git a/rules/network/command_and_control_cobalt_strike_default_teamserver_cert.toml b/rules/network/command_and_control_cobalt_strike_default_teamserver_cert.toml index a7df6f7a2..066bf738b 100644 --- a/rules/network/command_and_control_cobalt_strike_default_teamserver_cert.toml +++ b/rules/network/command_and_control_cobalt_strike_default_teamserver_cert.toml @@ -1,6 +1,6 @@ [metadata] creation_date = "2020/10/05" -integration = ["endpoint"] +integration = ["network_traffic"] maturity = "production" min_stack_comments = "New fields added: required_fields, related_integrations, setup" min_stack_version = "8.3.0" @@ -16,7 +16,7 @@ SHA256 hashing algorithms (the default is SHA1). See the References section for configuration. """ from = "now-9m" -index = ["auditbeat-*", "filebeat-*", "packetbeat-*", "logs-endpoint.events.*"] +index = ["packetbeat-*", "logs-network_traffic.*"] language = "kuery" license = "Elastic License v2" name = "Default Cobalt Strike Team Server Certificate" @@ -39,7 +39,7 @@ timestamp_override = "event.ingested" type = "query" query = ''' -event.category:(network or network_traffic) and (tls.server.hash.md5:950098276A495286EB2A2556FBAB6D83 or +event.dataset: network_traffic.tls and (tls.server.hash.md5:950098276A495286EB2A2556FBAB6D83 or tls.server.hash.sha1:6ECE5ECE4192683D2D84E25B0BA7E04F9CB7EB7C or tls.server.hash.sha256:87F2085C32B6A2CC709B365F55873E207A9CAA10BFFECF2FD16D3CF9D94D390C) ''' diff --git a/rules/network/command_and_control_download_rar_powershell_from_internet.toml b/rules/network/command_and_control_download_rar_powershell_from_internet.toml index ee28e0f59..7c33cbc33 100644 --- a/rules/network/command_and_control_download_rar_powershell_from_internet.toml +++ b/rules/network/command_and_control_download_rar_powershell_from_internet.toml @@ -1,6 +1,6 @@ [metadata] creation_date = "2020/07/02" -integration = ["endpoint"] +integration = ["network_traffic"] maturity = "production" min_stack_comments = "New fields added: required_fields, related_integrations, setup" min_stack_version = "8.3.0" @@ -21,7 +21,7 @@ false_positives = [ """, ] from = "now-9m" -index = ["auditbeat-*", "filebeat-*", "packetbeat-*", "logs-endpoint.events.*"] +index = ["packetbeat-*", "logs-network_traffic.*"] language = "kuery" license = "Elastic License v2" name = "Roshal Archive (RAR) or PowerShell File Downloaded from the Internet" @@ -41,7 +41,7 @@ timestamp_override = "event.ingested" type = "query" query = ''' -event.category:(network or network_traffic) and network.protocol:http and +event.dataset: (network_traffic.http or network_traffic.tls) and (url.extension:(ps1 or rar) or url.path:(*.ps1 or *.rar)) and not destination.ip:( 10.0.0.0/8 or diff --git a/rules/network/command_and_control_fin7_c2_behavior.toml b/rules/network/command_and_control_fin7_c2_behavior.toml index a2eca9823..3d1a895ca 100644 --- a/rules/network/command_and_control_fin7_c2_behavior.toml +++ b/rules/network/command_and_control_fin7_c2_behavior.toml @@ -1,6 +1,6 @@ [metadata] creation_date = "2020/07/06" -integration = ["endpoint"] +integration = ["network_traffic"] maturity = "production" min_stack_comments = "New fields added: required_fields, related_integrations, setup" min_stack_version = "8.3.0" @@ -19,7 +19,7 @@ false_positives = [ """, ] from = "now-9m" -index = ["auditbeat-*", "filebeat-*", "packetbeat-*", "logs-endpoint.events.*"] +index = ["packetbeat-*", "logs-network_traffic.*"] language = "lucene" license = "Elastic License v2" name = "Possible FIN7 DGA Command and Control Behavior" @@ -37,8 +37,8 @@ timestamp_override = "event.ingested" type = "query" query = ''' -event.category:(network OR network_traffic) AND type:(tls OR http) AND network.transport:tcp -AND destination.domain:/[a-zA-Z]{4,5}\.(pw|us|club|info|site|top)/ AND NOT destination.domain:zoom.us +event.dataset: (network_traffic.tls or network_traffic.http) AND +destination.domain:/[a-zA-Z]{4,5}\.(pw|us|club|info|site|top)/ AND NOT destination.domain:zoom.us ''' diff --git a/rules/network/command_and_control_halfbaked_beacon.toml b/rules/network/command_and_control_halfbaked_beacon.toml index a07f1b473..9eaf40abd 100644 --- a/rules/network/command_and_control_halfbaked_beacon.toml +++ b/rules/network/command_and_control_halfbaked_beacon.toml @@ -1,6 +1,6 @@ [metadata] creation_date = "2020/07/06" -integration = ["endpoint"] +integration = ["network_traffic"] maturity = "production" min_stack_comments = "New fields added: required_fields, related_integrations, setup" min_stack_version = "8.3.0" @@ -19,7 +19,7 @@ false_positives = [ """, ] from = "now-9m" -index = ["auditbeat-*", "filebeat-*", "packetbeat-*", "logs-endpoint.events.*"] +index = ["packetbeat-*", "logs-network_traffic.*"] language = "lucene" license = "Elastic License v2" name = "Halfbaked Command and Control Beacon" @@ -38,7 +38,7 @@ timestamp_override = "event.ingested" type = "query" query = ''' -event.category:(network OR network_traffic) AND network.protocol:http AND +event.dataset: (network_traffic.tls or network_traffic.http) AND network.transport:tcp AND url.full:/http:\/\/[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}\/cd/ AND destination.port:(53 OR 80 OR 8080 OR 443) ''' diff --git a/rules/network/command_and_control_nat_traversal_port_activity.toml b/rules/network/command_and_control_nat_traversal_port_activity.toml index abca787bf..74fff7e13 100644 --- a/rules/network/command_and_control_nat_traversal_port_activity.toml +++ b/rules/network/command_and_control_nat_traversal_port_activity.toml @@ -1,6 +1,6 @@ [metadata] creation_date = "2020/02/18" -integration = ["endpoint"] +integration = ["network_traffic"] maturity = "production" min_stack_comments = "New fields added: required_fields, related_integrations, setup" min_stack_version = "8.3.0" @@ -23,7 +23,7 @@ false_positives = [ """, ] from = "now-9m" -index = ["auditbeat-*", "filebeat-*", "packetbeat-*", "logs-endpoint.events.*"] +index = ["packetbeat-*", "logs-network_traffic.*"] language = "kuery" license = "Elastic License v2" name = "IPSEC NAT Traversal Port Activity" @@ -35,7 +35,7 @@ timestamp_override = "event.ingested" type = "query" query = ''' -event.category:(network or network_traffic) and network.transport:udp and destination.port:4500 +event.dataset: network_traffic.flow and network.transport:udp and destination.port:4500 ''' diff --git a/rules/network/command_and_control_port_26_activity.toml b/rules/network/command_and_control_port_26_activity.toml index ffc9bbe40..1d460d272 100644 --- a/rules/network/command_and_control_port_26_activity.toml +++ b/rules/network/command_and_control_port_26_activity.toml @@ -1,6 +1,6 @@ [metadata] creation_date = "2020/02/18" -integration = ["endpoint"] +integration = ["network_traffic"] maturity = "production" min_stack_comments = "New fields added: required_fields, related_integrations, setup" min_stack_version = "8.3.0" @@ -20,7 +20,7 @@ false_positives = [ """, ] from = "now-9m" -index = ["auditbeat-*", "filebeat-*", "packetbeat-*", "logs-endpoint.events.*"] +index = ["packetbeat-*", "logs-network_traffic.*"] language = "kuery" license = "Elastic License v2" name = "SMTP on Port 26/TCP" @@ -36,7 +36,7 @@ timestamp_override = "event.ingested" type = "query" query = ''' -event.category:(network or network_traffic) and network.transport:tcp and (destination.port:26 or (event.dataset:zeek.smtp and destination.port:26)) +event.dataset: network_traffic.flow and network.transport:tcp and (destination.port:26 or (event.dataset:zeek.smtp and destination.port:26)) ''' diff --git a/rules/network/command_and_control_rdp_remote_desktop_protocol_from_the_internet.toml b/rules/network/command_and_control_rdp_remote_desktop_protocol_from_the_internet.toml index 4ce6a4dc5..4fd9b1e3d 100644 --- a/rules/network/command_and_control_rdp_remote_desktop_protocol_from_the_internet.toml +++ b/rules/network/command_and_control_rdp_remote_desktop_protocol_from_the_internet.toml @@ -1,6 +1,6 @@ [metadata] creation_date = "2020/02/18" -integration = ["endpoint"] +integration = ["network_traffic"] maturity = "production" min_stack_comments = "New fields added: required_fields, related_integrations, setup" min_stack_version = "8.3.0" @@ -25,7 +25,7 @@ false_positives = [ """, ] from = "now-9m" -index = ["auditbeat-*", "filebeat-*", "packetbeat-*", "logs-endpoint.events.*"] +index = ["packetbeat-*", "logs-network_traffic.*"] language = "kuery" license = "Elastic License v2" name = "RDP (Remote Desktop Protocol) from the Internet" @@ -40,7 +40,7 @@ timestamp_override = "event.ingested" type = "query" query = ''' -event.category:(network or network_traffic) and network.transport:tcp and (destination.port:3389 or event.dataset:zeek.rdp) and +event.dataset: network_traffic.flow and network.transport:tcp and (destination.port:3389 or event.dataset:zeek.rdp) and not source.ip:( 10.0.0.0/8 or 127.0.0.0/8 or diff --git a/rules/network/command_and_control_vnc_virtual_network_computing_from_the_internet.toml b/rules/network/command_and_control_vnc_virtual_network_computing_from_the_internet.toml index 6156260d3..afd295d9a 100644 --- a/rules/network/command_and_control_vnc_virtual_network_computing_from_the_internet.toml +++ b/rules/network/command_and_control_vnc_virtual_network_computing_from_the_internet.toml @@ -1,6 +1,6 @@ [metadata] creation_date = "2020/02/18" -integration = ["endpoint"] +integration = ["network_traffic"] maturity = "production" min_stack_comments = "New fields added: required_fields, related_integrations, setup" min_stack_version = "8.3.0" @@ -23,7 +23,7 @@ false_positives = [ """, ] from = "now-9m" -index = ["auditbeat-*", "filebeat-*", "packetbeat-*", "logs-endpoint.events.*"] +index = ["packetbeat-*", "logs-network_traffic.*"] language = "kuery" license = "Elastic License v2" name = "VNC (Virtual Network Computing) from the Internet" @@ -36,7 +36,7 @@ timestamp_override = "event.ingested" type = "query" query = ''' -event.category:(network or network_traffic) and network.transport:tcp and destination.port >= 5800 and destination.port <= 5810 and +event.dataset: network_traffic.flow and network.transport:tcp and destination.port >= 5800 and destination.port <= 5810 and not source.ip:( 10.0.0.0/8 or 127.0.0.0/8 or diff --git a/rules/network/command_and_control_vnc_virtual_network_computing_to_the_internet.toml b/rules/network/command_and_control_vnc_virtual_network_computing_to_the_internet.toml index 067d07809..adb6ed821 100644 --- a/rules/network/command_and_control_vnc_virtual_network_computing_to_the_internet.toml +++ b/rules/network/command_and_control_vnc_virtual_network_computing_to_the_internet.toml @@ -1,6 +1,6 @@ [metadata] creation_date = "2020/02/18" -integration = ["endpoint"] +integration = ["network_traffic"] maturity = "production" min_stack_comments = "New fields added: required_fields, related_integrations, setup" min_stack_version = "8.3.0" @@ -23,7 +23,7 @@ false_positives = [ """, ] from = "now-9m" -index = ["auditbeat-*", "filebeat-*", "packetbeat-*", "logs-endpoint.events.*"] +index = ["packetbeat-*", "logs-network_traffic.*"] language = "kuery" license = "Elastic License v2" name = "VNC (Virtual Network Computing) to the Internet" @@ -36,7 +36,7 @@ timestamp_override = "event.ingested" type = "query" query = ''' -event.category:(network or network_traffic) and network.transport:tcp and destination.port >= 5800 and destination.port <= 5810 and +event.dataset: network_traffic.flow and network.transport:tcp and destination.port >= 5800 and destination.port <= 5810 and source.ip:( 10.0.0.0/8 or 172.16.0.0/12 or diff --git a/rules/network/initial_access_rpc_remote_procedure_call_from_the_internet.toml b/rules/network/initial_access_rpc_remote_procedure_call_from_the_internet.toml index d3c3cde8a..091203ffc 100644 --- a/rules/network/initial_access_rpc_remote_procedure_call_from_the_internet.toml +++ b/rules/network/initial_access_rpc_remote_procedure_call_from_the_internet.toml @@ -1,6 +1,6 @@ [metadata] creation_date = "2020/02/18" -integration = ["endpoint"] +integration = ["network_traffic"] maturity = "production" min_stack_comments = "New fields added: required_fields, related_integrations, setup" min_stack_version = "8.3.0" @@ -15,7 +15,7 @@ directly exposed to the Internet, as it is frequently targeted and exploited by backdoor vector. """ from = "now-9m" -index = ["auditbeat-*", "filebeat-*", "packetbeat-*", "logs-endpoint.events.*"] +index = ["packetbeat-*", "logs-network_traffic.*"] language = "kuery" license = "Elastic License v2" name = "RPC (Remote Procedure Call) from the Internet" @@ -28,7 +28,7 @@ timestamp_override = "event.ingested" type = "query" query = ''' -event.category:(network or network_traffic) and network.transport:tcp and (destination.port:135 or event.dataset:zeek.dce_rpc) and +event.dataset: network_traffic.flow and network.transport:tcp and (destination.port:135 or event.dataset:zeek.dce_rpc) and not source.ip:( 10.0.0.0/8 or 127.0.0.0/8 or diff --git a/rules/network/initial_access_rpc_remote_procedure_call_to_the_internet.toml b/rules/network/initial_access_rpc_remote_procedure_call_to_the_internet.toml index 6603b6e26..7a43eecdb 100644 --- a/rules/network/initial_access_rpc_remote_procedure_call_to_the_internet.toml +++ b/rules/network/initial_access_rpc_remote_procedure_call_to_the_internet.toml @@ -1,6 +1,6 @@ [metadata] creation_date = "2020/02/18" -integration = ["endpoint"] +integration = ["network_traffic"] maturity = "production" min_stack_comments = "New fields added: required_fields, related_integrations, setup" min_stack_version = "8.3.0" @@ -15,7 +15,7 @@ directly exposed to the Internet, as it is frequently targeted and exploited by backdoor vector. """ from = "now-9m" -index = ["auditbeat-*", "filebeat-*", "packetbeat-*", "logs-endpoint.events.*"] +index = ["packetbeat-*", "logs-network_traffic.*"] language = "kuery" license = "Elastic License v2" name = "RPC (Remote Procedure Call) to the Internet" @@ -28,7 +28,7 @@ timestamp_override = "event.ingested" type = "query" query = ''' -event.category:(network or network_traffic) and network.transport:tcp and (destination.port:135 or event.dataset:zeek.dce_rpc) and +event.dataset: network_traffic.flow and network.transport:tcp and (destination.port:135 or event.dataset:zeek.dce_rpc) and source.ip:( 10.0.0.0/8 or 172.16.0.0/12 or diff --git a/rules/network/initial_access_smb_windows_file_sharing_activity_to_the_internet.toml b/rules/network/initial_access_smb_windows_file_sharing_activity_to_the_internet.toml index 5425c5657..9cb466a12 100644 --- a/rules/network/initial_access_smb_windows_file_sharing_activity_to_the_internet.toml +++ b/rules/network/initial_access_smb_windows_file_sharing_activity_to_the_internet.toml @@ -1,6 +1,6 @@ [metadata] creation_date = "2020/02/18" -integration = ["endpoint"] +integration = ["network_traffic"] maturity = "production" min_stack_comments = "New fields added: required_fields, related_integrations, setup" min_stack_version = "8.3.0" @@ -15,7 +15,7 @@ systems. It should almost never be directly exposed to the Internet, as it is fr threat actors as an initial access or backdoor vector or for data exfiltration. """ from = "now-9m" -index = ["auditbeat-*", "filebeat-*", "packetbeat-*", "logs-endpoint.events.*"] +index = ["packetbeat-*", "logs-network_traffic.*"] language = "kuery" license = "Elastic License v2" name = "SMB (Windows File Sharing) Activity to the Internet" @@ -28,7 +28,7 @@ timestamp_override = "event.ingested" type = "query" query = ''' -event.category:(network or network_traffic) and network.transport:tcp and (destination.port:(139 or 445) or event.dataset:zeek.smb) and +event.dataset: network_traffic.flow and network.transport:tcp and (destination.port:(139 or 445) or event.dataset:zeek.smb) and source.ip:( 10.0.0.0/8 or 172.16.0.0/12 or diff --git a/rules/network/initial_access_unsecure_elasticsearch_node.toml b/rules/network/initial_access_unsecure_elasticsearch_node.toml index c61905524..b1fa05a1f 100644 --- a/rules/network/initial_access_unsecure_elasticsearch_node.toml +++ b/rules/network/initial_access_unsecure_elasticsearch_node.toml @@ -1,6 +1,6 @@ [metadata] creation_date = "2020/08/11" -integration = ["endpoint"] +integration = ["network_traffic"] maturity = "production" min_stack_comments = "New fields added: required_fields, related_integrations, setup" min_stack_version = "8.3.0" @@ -19,7 +19,7 @@ false_positives = [ """, ] from = "now-9m" -index = ["auditbeat-*", "filebeat-*", "packetbeat-*", "logs-endpoint.events.*"] +index = ["packetbeat-*", "logs-network_traffic.*"] language = "lucene" license = "Elastic License v2" name = "Inbound Connection to an Unsecure Elasticsearch Node" @@ -38,7 +38,7 @@ timestamp_override = "event.ingested" type = "query" query = ''' -event.category:network_traffic AND network.protocol:http AND status:OK AND destination.port:9200 AND network.direction:inbound AND NOT http.response.headers.content-type:"image/x-icon" AND NOT _exists_:http.request.headers.authorization +event.dataset: network_traffic.http AND status:OK AND destination.port:9200 AND network.direction:inbound AND NOT http.response.headers.content-type:"image/x-icon" AND NOT _exists_:http.request.headers.authorization ''' diff --git a/rules/network/lateral_movement_dns_server_overflow.toml b/rules/network/lateral_movement_dns_server_overflow.toml index cf80f59bd..14809d8b8 100644 --- a/rules/network/lateral_movement_dns_server_overflow.toml +++ b/rules/network/lateral_movement_dns_server_overflow.toml @@ -1,5 +1,6 @@ [metadata] creation_date = "2020/07/16" +integration = ["network_traffic"] maturity = "production" min_stack_comments = "New fields added: required_fields, related_integrations, setup" min_stack_version = "8.3.0" @@ -18,7 +19,7 @@ false_positives = [ authorized vulnerability scan or compromise assessment. """, ] -index = ["packetbeat-*", "filebeat-*"] +index = ["packetbeat-*", "logs-network_traffic.*"] language = "kuery" license = "Elastic License v2" name = "Abnormally Large DNS Response" @@ -68,7 +69,7 @@ timestamp_override = "event.ingested" type = "query" query = ''' -event.category:(network or network_traffic) and destination.port:53 and +event.dataset: network_traffic.dns and (event.dataset:zeek.dns or type:dns or event.type:connection) and network.bytes > 60000 '''