From e0f2e8b4a9dbc8315c709132ebc09b96752c0038 Mon Sep 17 00:00:00 2001
From: Andrew Pease <7442091+peasead@users.noreply.github.com>
Date: Wed, 8 Jul 2020 14:19:35 -0500
Subject: [PATCH] Add dataset and index to network rules (#15)

* Add dataset and index to network rules
* Restore iptables changes
* Fix beats parsing logic
* Updated date and ECS version
* Only update modules if empty

Co-authored-by: Justin Ibarra <brokensound77@users.noreply.github.com>
Co-authored-by: Ross Wolf <31489089+rw-access@users.noreply.github.com>
---
 detection_rules/beats.py                      | 27 ++++++++++++++-----
 ..._control_dns_directly_to_the_internet.toml | 13 +++++----
 ...fer_protocol_activity_to_the_internet.toml |  9 +++----
 ...hat_protocol_activity_to_the_internet.toml |  9 +++----
 ...d_control_nat_traversal_port_activity.toml |  9 +++----
 .../command_and_control_port_26_activity.toml |  9 +++----
 ...ol_port_8000_activity_to_the_internet.toml |  9 +++----
 ..._to_point_tunneling_protocol_activity.toml |  9 +++----
 ...l_proxy_port_activity_to_the_internet.toml |  9 +++----
 ...te_desktop_protocol_from_the_internet.toml |  9 +++----
 ...mand_and_control_smtp_to_the_internet.toml |  9 +++----
 ..._server_port_activity_to_the_internet.toml |  9 +++----
 ...ol_ssh_secure_shell_from_the_internet.toml |  9 +++----
 ...trol_ssh_secure_shell_to_the_internet.toml |  9 +++----
 ...mand_and_control_telnet_port_activity.toml |  9 +++----
 ..._control_tor_activity_to_the_internet.toml |  9 +++----
 ...l_network_computing_from_the_internet.toml |  9 +++----
 ...ual_network_computing_to_the_internet.toml |  9 +++----
 ...mote_desktop_protocol_to_the_internet.toml |  9 +++----
 ...mote_procedure_call_from_the_internet.toml |  9 +++----
 ...remote_procedure_call_to_the_internet.toml |  9 +++----
 ...file_sharing_activity_to_the_internet.toml |  9 +++----
 22 files changed, 107 insertions(+), 113 deletions(-)

diff --git a/detection_rules/beats.py b/detection_rules/beats.py
index 316f8ae6e..3ce5ca2e6 100644
--- a/detection_rules/beats.py
+++ b/detection_rules/beats.py
@@ -72,8 +72,11 @@ def _flatten_schema(schema: list, prefix="") -> list:
             flattened.extend(_flatten_schema(s["fields"], prefix=prefix + s["name"] + "."))
         elif "fields" in s:
             flattened.extend(_flatten_schema(s["fields"], prefix=prefix))
-        elif "type" in s:
+        elif "name" in s and "description" in s:
             s = s.copy()
+            # type is implicitly keyword if not defined
+            # example: https://github.com/elastic/beats/blob/master/packetbeat/_meta/fields.common.yml#L7-L12
+            s.setdefault("type", "keyword")
             s["name"] = prefix + s["name"]
             flattened.append(s)
 
@@ -93,16 +96,23 @@ def get_field_schema(base_directory, prefix="", include_common=False):
     return flattened
 
 
-def get_beats_schema(schema: dict, beat: str, module: str, *datasets: str):
+def get_beat_root_schema(schema: dict, beat: str):
+    if beat not in schema:
+        raise KeyError(f"Unknown beats module {beat}")
+
+    beat_dir = schema[beat]
+    flattened = get_field_schema(beat_dir, include_common=True)
+
+    return {field["name"]: field for field in sorted(flattened, key=lambda f: f["name"])}
+
+
+def get_beats_sub_schema(schema: dict, beat: str, module: str, *datasets: str):
     if beat not in schema:
         raise KeyError(f"Unknown beats module {beat}")
 
     flattened = []
     beat_dir = schema[beat]
-    flattened.extend(get_field_schema(beat_dir, include_common=True))
-
     module_dir = beat_dir.get("folders", {}).get("module", {}).get("folders", {}).get(module, {})
-    flattened.extend(get_field_schema(module_dir, include_common=True))
 
     # if we only have a module then we'll work with what we got
     if not datasets:
@@ -149,12 +159,17 @@ def get_schema_for_query(tree: kql.ast, beats: list) -> dict:
 
     beats_schema = read_beats_schema()
 
+    # infer the module if only a dataset are defined
+    if not modules:
+        modules.update(ds.split(".")[0] for ds in datasets if "." in ds)
+
     for beat in beats:
         # if no modules are specified then grab them all
         # all_modules = list(beats_schema.get(beat, {}).get("folders", {}).get("module", {}).get("folders", {}))
         # beat_modules = modules or all_modules
+        filtered.update(get_beat_root_schema(beats_schema, beat))
 
         for module in modules:
-            filtered.update(get_beats_schema(beats_schema, beat, module, *datasets))
+            filtered.update(get_beats_sub_schema(beats_schema, beat, module, *datasets))
 
     return filtered
diff --git a/rules/network/command_and_control_dns_directly_to_the_internet.toml b/rules/network/command_and_control_dns_directly_to_the_internet.toml
index e4759fea0..ae1a0968a 100644
--- a/rules/network/command_and_control_dns_directly_to_the_internet.toml
+++ b/rules/network/command_and_control_dns_directly_to_the_internet.toml
@@ -1,8 +1,8 @@
 [metadata]
 creation_date = "2020/02/18"
-ecs_version = ["1.4.0"]
+ecs_version = ["1.5.0"]
 maturity = "production"
-updated_date = "2020/03/09"
+updated_date = "2020/07/02"
 
 [rule]
 author = ["Elastic"]
@@ -21,7 +21,7 @@ false_positives = [
     destinations. In that case, such devices or networks can be excluded from this rule when this is expected behavior.
     """,
 ]
-index = ["filebeat-*"]
+index = ["filebeat-*", "packetbeat-*"]
 language = "kuery"
 license = "Elastic License"
 name = "DNS Activity to the Internet"
@@ -36,9 +36,9 @@ tags = ["Elastic", "Network"]
 type = "query"
 
 query = '''
-destination.port:53 and source.ip:(10.0.0.0/8 or 172.16.0.0/12 or 192.168.0.0/16) and
-  not destination.ip:(10.0.0.0/8 or 127.0.0.0/8 or 169.254.169.254/32 or 172.16.0.0/12 or
-                        192.168.0.0/16 or 224.0.0.251 or 224.0.0.252 or 255.255.255.255 or "::1" or "ff02::fb")
+event.category:(network or network_traffic) and (event.type:connection or type:dns) and (destination.port:53 or event.dataset:zeek.dns) 
+and source.ip:(10.0.0.0/8 or 172.16.0.0/12 or 192.168.0.0/16) and not destination.ip:(10.0.0.0/8 or 127.0.0.0/8 or 169.254.169.254/32 or
+  172.16.0.0/12 or 192.168.0.0/16 or 224.0.0.251 or 224.0.0.252 or 255.255.255.255 or "::1" or "ff02::fb")
 '''
 
 
@@ -54,4 +54,3 @@ reference = "https://attack.mitre.org/techniques/T1043/"
 id = "TA0011"
 name = "Command and Control"
 reference = "https://attack.mitre.org/tactics/TA0011/"
-
diff --git a/rules/network/command_and_control_ftp_file_transfer_protocol_activity_to_the_internet.toml b/rules/network/command_and_control_ftp_file_transfer_protocol_activity_to_the_internet.toml
index cee95fda2..afcbb5416 100644
--- a/rules/network/command_and_control_ftp_file_transfer_protocol_activity_to_the_internet.toml
+++ b/rules/network/command_and_control_ftp_file_transfer_protocol_activity_to_the_internet.toml
@@ -1,8 +1,8 @@
 [metadata]
 creation_date = "2020/02/18"
-ecs_version = ["1.4.0"]
+ecs_version = ["1.5.0"]
 maturity = "production"
-updated_date = "2020/03/09"
+updated_date = "2020/07/02"
 
 [rule]
 author = ["Elastic"]
@@ -22,7 +22,7 @@ false_positives = [
     server that has no known associated FTP workflow or business requirement is often suspicious.
     """,
 ]
-index = ["filebeat-*"]
+index = ["filebeat-*", "packetbeat-*"]
 language = "kuery"
 license = "Elastic License"
 name = "FTP (File Transfer Protocol) Activity to the Internet"
@@ -33,7 +33,7 @@ tags = ["Elastic", "Network"]
 type = "query"
 
 query = '''
-network.transport:tcp and destination.port:(20 or 21) and
+event.category:(network or network_traffic) and network.transport:tcp and (destination.port:(20 or 21) or event.dataset:zeek.ftp) and
    source.ip:(10.0.0.0/8 or 172.16.0.0/12 or 192.168.0.0/16) and
    not destination.ip:(10.0.0.0/8 or 127.0.0.0/8 or 172.16.0.0/12 or 192.168.0.0/16 or "::1")
 '''
@@ -63,4 +63,3 @@ reference = "https://attack.mitre.org/techniques/T1048/"
 id = "TA0010"
 name = "Exfiltration"
 reference = "https://attack.mitre.org/tactics/TA0010/"
-
diff --git a/rules/network/command_and_control_irc_internet_relay_chat_protocol_activity_to_the_internet.toml b/rules/network/command_and_control_irc_internet_relay_chat_protocol_activity_to_the_internet.toml
index e48d22395..09ec11d31 100644
--- a/rules/network/command_and_control_irc_internet_relay_chat_protocol_activity_to_the_internet.toml
+++ b/rules/network/command_and_control_irc_internet_relay_chat_protocol_activity_to_the_internet.toml
@@ -1,8 +1,8 @@
 [metadata]
 creation_date = "2020/02/18"
-ecs_version = ["1.4.0"]
+ecs_version = ["1.5.0"]
 maturity = "production"
-updated_date = "2020/03/09"
+updated_date = "2020/07/02"
 
 [rule]
 author = ["Elastic"]
@@ -21,7 +21,7 @@ false_positives = [
     and usually only appears in local traffic using private IPs, which does not match this rule's conditions.
     """,
 ]
-index = ["filebeat-*"]
+index = ["filebeat-*", "packetbeat-*"]
 language = "kuery"
 license = "Elastic License"
 name = "IRC (Internet Relay Chat) Protocol Activity to the Internet"
@@ -32,7 +32,7 @@ tags = ["Elastic", "Network"]
 type = "query"
 
 query = '''
-network.transport:tcp and destination.port:(6667 or 6697) and
+event.category:(network or network_traffic) and network.transport:tcp and (destination.port:(6667 or 6697) or event.dataset:zeek.irc) and
    source.ip:(10.0.0.0/8 or 172.16.0.0/12 or 192.168.0.0/16) and
    not destination.ip:(10.0.0.0/8 or 127.0.0.0/8 or 172.16.0.0/12 or 192.168.0.0/16 or "::1")
 '''
@@ -62,4 +62,3 @@ reference = "https://attack.mitre.org/techniques/T1048/"
 id = "TA0010"
 name = "Exfiltration"
 reference = "https://attack.mitre.org/tactics/TA0011/"
-
diff --git a/rules/network/command_and_control_nat_traversal_port_activity.toml b/rules/network/command_and_control_nat_traversal_port_activity.toml
index cd7a1e31e..1818d2ed8 100644
--- a/rules/network/command_and_control_nat_traversal_port_activity.toml
+++ b/rules/network/command_and_control_nat_traversal_port_activity.toml
@@ -1,8 +1,8 @@
 [metadata]
 creation_date = "2020/02/18"
-ecs_version = ["1.4.0"]
+ecs_version = ["1.5.0"]
 maturity = "production"
-updated_date = "2020/02/18"
+updated_date = "2020/07/02"
 
 [rule]
 author = ["Elastic"]
@@ -20,7 +20,7 @@ false_positives = [
     port in the range by coincidence. This is uncommon but such servers can be excluded.
     """,
 ]
-index = ["filebeat-*"]
+index = ["filebeat-*", "packetbeat-*"]
 language = "kuery"
 license = "Elastic License"
 name = "IPSEC NAT Traversal Port Activity"
@@ -31,7 +31,7 @@ tags = ["Elastic", "Network"]
 type = "query"
 
 query = '''
-network.transport:udp and destination.port:4500
+event.category:(network or network_traffic) and network.transport:udp and destination.port:4500
 '''
 
 
@@ -47,4 +47,3 @@ reference = "https://attack.mitre.org/techniques/T1043/"
 id = "TA0011"
 name = "Command and Control"
 reference = "https://attack.mitre.org/tactics/TA0011/"
-
diff --git a/rules/network/command_and_control_port_26_activity.toml b/rules/network/command_and_control_port_26_activity.toml
index 9f7ea251c..41d2fbf0f 100644
--- a/rules/network/command_and_control_port_26_activity.toml
+++ b/rules/network/command_and_control_port_26_activity.toml
@@ -1,8 +1,8 @@
 [metadata]
 creation_date = "2020/02/18"
-ecs_version = ["1.4.0"]
+ecs_version = ["1.5.0"]
 maturity = "production"
-updated_date = "2020/02/18"
+updated_date = "2020/07/02"
 
 [rule]
 author = ["Elastic"]
@@ -17,7 +17,7 @@ false_positives = [
     expected behavior.
     """,
 ]
-index = ["filebeat-*"]
+index = ["filebeat-*", "packetbeat-*"]
 language = "kuery"
 license = "Elastic License"
 name = "SMTP on Port 26/TCP"
@@ -32,7 +32,7 @@ tags = ["Elastic", "Network"]
 type = "query"
 
 query = '''
-network.transport:tcp and destination.port:26
+event.category:(network or network_traffic) and network.transport:tcp and (destination.port:26 or (event.dataset:zeek.smtp and destination.port:26))
 '''
 
 
@@ -60,4 +60,3 @@ reference = "https://attack.mitre.org/techniques/T1048/"
 id = "TA0010"
 name = "Exfiltration"
 reference = "https://attack.mitre.org/tactics/TA0011/"
-
diff --git a/rules/network/command_and_control_port_8000_activity_to_the_internet.toml b/rules/network/command_and_control_port_8000_activity_to_the_internet.toml
index 9ace7be07..bbec1485c 100644
--- a/rules/network/command_and_control_port_8000_activity_to_the_internet.toml
+++ b/rules/network/command_and_control_port_8000_activity_to_the_internet.toml
@@ -1,8 +1,8 @@
 [metadata]
 creation_date = "2020/02/18"
-ecs_version = ["1.4.0"]
+ecs_version = ["1.5.0"]
 maturity = "production"
-updated_date = "2020/03/09"
+updated_date = "2020/07/02"
 
 [rule]
 author = ["Elastic"]
@@ -20,7 +20,7 @@ false_positives = [
     this port when VPNs or direct connects are not in use and cloud instances are accessed across the Internet.
     """,
 ]
-index = ["filebeat-*"]
+index = ["filebeat-*", "packetbeat-*"]
 language = "kuery"
 license = "Elastic License"
 name = "TCP Port 8000 Activity to the Internet"
@@ -31,7 +31,7 @@ tags = ["Elastic", "Network"]
 type = "query"
 
 query = '''
-network.transport:tcp and destination.port:8000 and
+event.category:(network or network_traffic) and network.transport:tcp and destination.port:8000 and
   source.ip:(10.0.0.0/8 or 172.16.0.0/12 or 192.168.0.0/16) and
   not destination.ip:(10.0.0.0/8 or 127.0.0.0/8 or 172.16.0.0/12 or 192.168.0.0/16 or "::1")
 '''
@@ -49,4 +49,3 @@ reference = "https://attack.mitre.org/techniques/T1043/"
 id = "TA0011"
 name = "Command and Control"
 reference = "https://attack.mitre.org/tactics/TA0011/"
-
diff --git a/rules/network/command_and_control_pptp_point_to_point_tunneling_protocol_activity.toml b/rules/network/command_and_control_pptp_point_to_point_tunneling_protocol_activity.toml
index f84f7ee36..81c863114 100644
--- a/rules/network/command_and_control_pptp_point_to_point_tunneling_protocol_activity.toml
+++ b/rules/network/command_and_control_pptp_point_to_point_tunneling_protocol_activity.toml
@@ -1,8 +1,8 @@
 [metadata]
 creation_date = "2020/02/18"
-ecs_version = ["1.4.0"]
+ecs_version = ["1.5.0"]
 maturity = "production"
-updated_date = "2020/02/18"
+updated_date = "2020/07/02"
 
 [rule]
 author = ["Elastic"]
@@ -19,7 +19,7 @@ false_positives = [
     be excluded.
     """,
 ]
-index = ["filebeat-*"]
+index = ["filebeat-*", "packetbeat-*"]
 language = "kuery"
 license = "Elastic License"
 name = "PPTP (Point to Point Tunneling Protocol) Activity"
@@ -30,7 +30,7 @@ tags = ["Elastic", "Network"]
 type = "query"
 
 query = '''
-network.transport:tcp and destination.port:1723
+event.category:(network or network_traffic) and network.transport:tcp and destination.port:1723
 '''
 
 
@@ -46,4 +46,3 @@ reference = "https://attack.mitre.org/techniques/T1043/"
 id = "TA0011"
 name = "Command and Control"
 reference = "https://attack.mitre.org/tactics/TA0011/"
-
diff --git a/rules/network/command_and_control_proxy_port_activity_to_the_internet.toml b/rules/network/command_and_control_proxy_port_activity_to_the_internet.toml
index 03dc4d93e..5a1adcdad 100644
--- a/rules/network/command_and_control_proxy_port_activity_to_the_internet.toml
+++ b/rules/network/command_and_control_proxy_port_activity_to_the_internet.toml
@@ -1,8 +1,8 @@
 [metadata]
 creation_date = "2020/02/18"
-ecs_version = ["1.4.0"]
+ecs_version = ["1.5.0"]
 maturity = "production"
-updated_date = "2020/03/09"
+updated_date = "2020/07/02"
 
 [rule]
 author = ["Elastic"]
@@ -23,7 +23,7 @@ false_positives = [
     the range by coincidence. In this case, such servers can be excluded if desired.
     """,
 ]
-index = ["filebeat-*"]
+index = ["filebeat-*", "packetbeat-*"]
 language = "kuery"
 license = "Elastic License"
 name = "Proxy Port Activity to the Internet"
@@ -34,7 +34,7 @@ tags = ["Elastic", "Network"]
 type = "query"
 
 query = '''
-network.transport:tcp and destination.port:(1080 or 3128 or 8080) and
+event.category:(network or network_traffic) and network.transport:tcp and (destination.port:(1080 or 3128 or 8080) or event.dataset:zeek.socks) and
   source.ip:(10.0.0.0/8 or 172.16.0.0/12 or 192.168.0.0/16) and
   not destination.ip:(10.0.0.0/8 or 127.0.0.0/8 or 172.16.0.0/12 or 192.168.0.0/16 or "::1")
 '''
@@ -52,4 +52,3 @@ reference = "https://attack.mitre.org/techniques/T1043/"
 id = "TA0011"
 name = "Command and Control"
 reference = "https://attack.mitre.org/tactics/TA0011/"
-
diff --git a/rules/network/command_and_control_rdp_remote_desktop_protocol_from_the_internet.toml b/rules/network/command_and_control_rdp_remote_desktop_protocol_from_the_internet.toml
index a8661453e..aab6c7e02 100644
--- a/rules/network/command_and_control_rdp_remote_desktop_protocol_from_the_internet.toml
+++ b/rules/network/command_and_control_rdp_remote_desktop_protocol_from_the_internet.toml
@@ -1,8 +1,8 @@
 [metadata]
 creation_date = "2020/02/18"
-ecs_version = ["1.4.0"]
+ecs_version = ["1.5.0"]
 maturity = "production"
-updated_date = "2020/03/09"
+updated_date = "2020/07/02"
 
 [rule]
 author = ["Elastic"]
@@ -22,7 +22,7 @@ false_positives = [
     not unexpected.
     """,
 ]
-index = ["filebeat-*"]
+index = ["filebeat-*", "packetbeat-*"]
 language = "kuery"
 license = "Elastic License"
 name = "RDP (Remote Desktop Protocol) from the Internet"
@@ -33,7 +33,7 @@ tags = ["Elastic", "Network"]
 type = "query"
 
 query = '''
-network.transport:tcp and destination.port:3389 and
+event.category:(network or network_traffic) and network.transport:tcp and (destination.port:3389 or event.dataset:zeek.rdp) and
   not source.ip:(10.0.0.0/8 or 172.16.0.0/12 or 192.168.0.0/16) and
   destination.ip:(10.0.0.0/8 or 127.0.0.0/8 or 172.16.0.0/12 or 192.168.0.0/16 or "::1")
 '''
@@ -75,4 +75,3 @@ reference = "https://attack.mitre.org/techniques/T1190/"
 id = "TA0001"
 name = "Initial Access"
 reference = "https://attack.mitre.org/tactics/TA0001/"
-
diff --git a/rules/network/command_and_control_smtp_to_the_internet.toml b/rules/network/command_and_control_smtp_to_the_internet.toml
index d3907563a..d6a4be6ae 100644
--- a/rules/network/command_and_control_smtp_to_the_internet.toml
+++ b/rules/network/command_and_control_smtp_to_the_internet.toml
@@ -1,8 +1,8 @@
 [metadata]
 creation_date = "2020/02/18"
-ecs_version = ["1.4.0"]
+ecs_version = ["1.5.0"]
 maturity = "production"
-updated_date = "2020/03/09"
+updated_date = "2020/07/02"
 
 [rule]
 author = ["Elastic"]
@@ -18,7 +18,7 @@ false_positives = [
     case, such devices or networks can be excluded from this rule if this is expected behavior.
     """,
 ]
-index = ["filebeat-*"]
+index = ["filebeat-*", "packetbeat-*"]
 language = "kuery"
 license = "Elastic License"
 name = "SMTP to the Internet"
@@ -29,7 +29,7 @@ tags = ["Elastic", "Network"]
 type = "query"
 
 query = '''
-network.transport:tcp and destination.port:(25 or 465 or 587) and
+event.category:(network or network_traffic) and network.transport:tcp and (destination.port:(25 or 465 or 587) or event.dataset:zeek.smtp) and
   source.ip:(10.0.0.0/8 or 172.16.0.0/12 or 192.168.0.0/16) and
   not destination.ip:(10.0.0.0/8 or 127.0.0.0/8 or 172.16.0.0/12 or 192.168.0.0/16 or "::1")
 '''
@@ -59,4 +59,3 @@ reference = "https://attack.mitre.org/techniques/T1048/"
 id = "TA0010"
 name = "Exfiltration"
 reference = "https://attack.mitre.org/tactics/TA0010/"
-
diff --git a/rules/network/command_and_control_sql_server_port_activity_to_the_internet.toml b/rules/network/command_and_control_sql_server_port_activity_to_the_internet.toml
index e40c2b4a2..9ed68e477 100644
--- a/rules/network/command_and_control_sql_server_port_activity_to_the_internet.toml
+++ b/rules/network/command_and_control_sql_server_port_activity_to_the_internet.toml
@@ -1,8 +1,8 @@
 [metadata]
 creation_date = "2020/02/18"
-ecs_version = ["1.4.0"]
+ecs_version = ["1.5.0"]
 maturity = "production"
-updated_date = "2020/07/01"
+updated_date = "2020/07/02"
 
 [rule]
 author = ["Elastic"]
@@ -19,7 +19,7 @@ false_positives = [
     database instances are accessed directly across the Internet.
     """,
 ]
-index = ["filebeat-*"]
+index = ["filebeat-*", "packetbeat-*"]
 language = "kuery"
 license = "Elastic License"
 name = "SQL Traffic to the Internet"
@@ -30,7 +30,7 @@ tags = ["Elastic", "Network"]
 type = "query"
 
 query = '''
-network.transport:tcp and destination.port:(1433 or 1521 or 3306 or 5432) and
+event.category:(network or network_traffic) and network.transport:tcp and (destination.port:(1433 or 1521 or 3306 or 5432) or event.dataset:zeek.mysql) and
   source.ip:(10.0.0.0/8 or 172.16.0.0/12 or 192.168.0.0/16) and
   not destination.ip:(10.0.0.0/8 or 127.0.0.0/8 or 172.16.0.0/12 or 192.168.0.0/16 or "::1")
 '''
@@ -48,4 +48,3 @@ reference = "https://attack.mitre.org/techniques/T1043/"
 id = "TA0011"
 name = "Command and Control"
 reference = "https://attack.mitre.org/tactics/TA0011/"
-
diff --git a/rules/network/command_and_control_ssh_secure_shell_from_the_internet.toml b/rules/network/command_and_control_ssh_secure_shell_from_the_internet.toml
index 0b19aaaca..5cd842bba 100644
--- a/rules/network/command_and_control_ssh_secure_shell_from_the_internet.toml
+++ b/rules/network/command_and_control_ssh_secure_shell_from_the_internet.toml
@@ -1,8 +1,8 @@
 [metadata]
 creation_date = "2020/02/18"
-ecs_version = ["1.4.0"]
+ecs_version = ["1.5.0"]
 maturity = "production"
-updated_date = "2020/03/09"
+updated_date = "2020/07/02"
 
 [rule]
 author = ["Elastic"]
@@ -22,7 +22,7 @@ false_positives = [
     not unexpected.
     """,
 ]
-index = ["filebeat-*"]
+index = ["filebeat-*", "packetbeat-*"]
 language = "kuery"
 license = "Elastic License"
 name = "SSH (Secure Shell) from the Internet"
@@ -33,7 +33,7 @@ tags = ["Elastic", "Network"]
 type = "query"
 
 query = '''
-network.transport:tcp and destination.port:22 and
+event.category:(network or network_traffic) and network.transport:tcp and (destination.port:22 or event.dataset:zeek.ssh) and
   not source.ip:(10.0.0.0/8 or 127.0.0.0/8 or 172.16.0.0/12 or 192.168.0.0/16 or "::1") and
   destination.ip:(10.0.0.0/8 or 172.16.0.0/12 or 192.168.0.0/16)
 '''
@@ -75,4 +75,3 @@ reference = "https://attack.mitre.org/techniques/T1190/"
 id = "TA0001"
 name = "Initial Access"
 reference = "https://attack.mitre.org/tactics/TA0001/"
-
diff --git a/rules/network/command_and_control_ssh_secure_shell_to_the_internet.toml b/rules/network/command_and_control_ssh_secure_shell_to_the_internet.toml
index 8317c02cb..733a2ea2a 100644
--- a/rules/network/command_and_control_ssh_secure_shell_to_the_internet.toml
+++ b/rules/network/command_and_control_ssh_secure_shell_to_the_internet.toml
@@ -1,8 +1,8 @@
 [metadata]
 creation_date = "2020/02/18"
-ecs_version = ["1.4.0"]
+ecs_version = ["1.5.0"]
 maturity = "production"
-updated_date = "2020/03/09"
+updated_date = "2020/07/02"
 
 [rule]
 author = ["Elastic"]
@@ -21,7 +21,7 @@ false_positives = [
     unexpected. Usage that is unfamiliar to server or network owners can be unexpected and suspicious.
     """,
 ]
-index = ["filebeat-*"]
+index = ["filebeat-*", "packetbeat-*"]
 language = "kuery"
 license = "Elastic License"
 name = "SSH (Secure Shell) to the Internet"
@@ -32,7 +32,7 @@ tags = ["Elastic", "Network"]
 type = "query"
 
 query = '''
-network.transport:tcp and destination.port:22 and
+event.category:(network or network_traffic) and network.transport:tcp and (destination.port:22 or event.dataset:zeek.ssh) and
   source.ip:(10.0.0.0/8 or 172.16.0.0/12 or 192.168.0.0/16) and
   not destination.ip:(10.0.0.0/8 or 127.0.0.0/8 or 172.16.0.0/12 or 192.168.0.0/16 or "::1")
 '''
@@ -50,4 +50,3 @@ reference = "https://attack.mitre.org/techniques/T1043/"
 id = "TA0011"
 name = "Command and Control"
 reference = "https://attack.mitre.org/tactics/TA0011/"
-
diff --git a/rules/network/command_and_control_telnet_port_activity.toml b/rules/network/command_and_control_telnet_port_activity.toml
index c9dffe729..cb01228c4 100644
--- a/rules/network/command_and_control_telnet_port_activity.toml
+++ b/rules/network/command_and_control_telnet_port_activity.toml
@@ -1,8 +1,8 @@
 [metadata]
 creation_date = "2020/02/18"
-ecs_version = ["1.4.0"]
+ecs_version = ["1.5.0"]
 maturity = "production"
-updated_date = "2020/02/18"
+updated_date = "2020/07/02"
 
 [rule]
 author = ["Elastic"]
@@ -21,7 +21,7 @@ false_positives = [
     server that has no known associated Telnet work-flow or business requirement is often suspicious.
     """,
 ]
-index = ["filebeat-*"]
+index = ["filebeat-*", "packetbeat-*"]
 language = "kuery"
 license = "Elastic License"
 name = "Telnet Port Activity"
@@ -32,7 +32,7 @@ tags = ["Elastic", "Network"]
 type = "query"
 
 query = '''
-network.transport:tcp and destination.port:23
+event.category:(network or network_traffic) and network.transport:tcp and destination.port:23
 '''
 
 
@@ -72,4 +72,3 @@ reference = "https://attack.mitre.org/techniques/T1190/"
 id = "TA0011"
 name = "Initial Access"
 reference = "https://attack.mitre.org/tactics/TA0011/"
-
diff --git a/rules/network/command_and_control_tor_activity_to_the_internet.toml b/rules/network/command_and_control_tor_activity_to_the_internet.toml
index a535f83df..c65541dd6 100644
--- a/rules/network/command_and_control_tor_activity_to_the_internet.toml
+++ b/rules/network/command_and_control_tor_activity_to_the_internet.toml
@@ -1,8 +1,8 @@
 [metadata]
 creation_date = "2020/02/18"
-ecs_version = ["1.4.0"]
+ecs_version = ["1.5.0"]
 maturity = "production"
-updated_date = "2020/03/09"
+updated_date = "2020/07/02"
 
 [rule]
 author = ["Elastic"]
@@ -19,7 +19,7 @@ false_positives = [
     this case, such servers can be excluded if desired.
     """,
 ]
-index = ["filebeat-*"]
+index = ["filebeat-*", "packetbeat-*"]
 language = "kuery"
 license = "Elastic License"
 name = "Tor Activity to the Internet"
@@ -30,7 +30,7 @@ tags = ["Elastic", "Network"]
 type = "query"
 
 query = '''
-network.transport:tcp and destination.port:(9001 or 9030) and
+event.category:(network or network_traffic) and network.transport:tcp and destination.port:(9001 or 9030) and
   source.ip:(10.0.0.0/8 or 172.16.0.0/12 or 192.168.0.0/16) and
   not destination.ip:(10.0.0.0/8 or 127.0.0.0/8 or 172.16.0.0/12 or 192.168.0.0/16 or "::1")
 '''
@@ -60,4 +60,3 @@ reference = "https://attack.mitre.org/techniques/T1188/"
 id = "TA0011"
 name = "Command and Control"
 reference = "https://attack.mitre.org/tactics/TA0011/"
-
diff --git a/rules/network/command_and_control_vnc_virtual_network_computing_from_the_internet.toml b/rules/network/command_and_control_vnc_virtual_network_computing_from_the_internet.toml
index de130929a..88a7705e7 100644
--- a/rules/network/command_and_control_vnc_virtual_network_computing_from_the_internet.toml
+++ b/rules/network/command_and_control_vnc_virtual_network_computing_from_the_internet.toml
@@ -1,8 +1,8 @@
 [metadata]
 creation_date = "2020/02/18"
-ecs_version = ["1.4.0"]
+ecs_version = ["1.5.0"]
 maturity = "production"
-updated_date = "2020/03/09"
+updated_date = "2020/07/02"
 
 [rule]
 author = ["Elastic"]
@@ -20,7 +20,7 @@ false_positives = [
     that is unfamiliar to server or network owners can be unexpected and suspicious.
     """,
 ]
-index = ["filebeat-*"]
+index = ["filebeat-*", "packetbeat-*"]
 language = "kuery"
 license = "Elastic License"
 name = "VNC (Virtual Network Computing) from the Internet"
@@ -31,7 +31,7 @@ tags = ["Elastic", "Network"]
 type = "query"
 
 query = '''
-network.transport:tcp and destination.port >= 5800 and destination.port <= 5810 and
+event.category:(network or network_traffic) and network.transport:tcp and destination.port >= 5800 and destination.port <= 5810 and
   not source.ip:(10.0.0.0/8 or 127.0.0.0/8 or 172.16.0.0/12 or 192.168.0.0/16 or "::1") and
   destination.ip:(10.0.0.0/8 or 172.16.0.0/12 or 192.168.0.0/16)
 '''
@@ -61,4 +61,3 @@ reference = "https://attack.mitre.org/techniques/T1190/"
 id = "TA0001"
 name = "Initial Access"
 reference = "https://attack.mitre.org/tactics/TA0001/"
-
diff --git a/rules/network/command_and_control_vnc_virtual_network_computing_to_the_internet.toml b/rules/network/command_and_control_vnc_virtual_network_computing_to_the_internet.toml
index f48467972..bd2fd6ae2 100644
--- a/rules/network/command_and_control_vnc_virtual_network_computing_to_the_internet.toml
+++ b/rules/network/command_and_control_vnc_virtual_network_computing_to_the_internet.toml
@@ -1,8 +1,8 @@
 [metadata]
 creation_date = "2020/02/18"
-ecs_version = ["1.4.0"]
+ecs_version = ["1.5.0"]
 maturity = "production"
-updated_date = "2020/03/09"
+updated_date = "2020/07/02"
 
 [rule]
 author = ["Elastic"]
@@ -20,7 +20,7 @@ false_positives = [
     that is unfamiliar to server or network owners can be unexpected and suspicious.
     """,
 ]
-index = ["filebeat-*"]
+index = ["filebeat-*", "packetbeat-*"]
 language = "kuery"
 license = "Elastic License"
 name = "VNC (Virtual Network Computing) to the Internet"
@@ -31,7 +31,7 @@ tags = ["Elastic", "Network"]
 type = "query"
 
 query = '''
-network.transport:tcp and destination.port >= 5800 and destination.port <= 5810 and
+event.category:(network or network_traffic) and network.transport:tcp and destination.port >= 5800 and destination.port <= 5810 and
   source.ip:(10.0.0.0/8 or 172.16.0.0/12 or 192.168.0.0/16) and
   not destination.ip:(10.0.0.0/8 or 127.0.0.0/8 or 172.16.0.0/12 or 192.168.0.0/16 or "::1")
 '''
@@ -49,4 +49,3 @@ reference = "https://attack.mitre.org/techniques/T1219/"
 id = "TA0011"
 name = "Command and Control"
 reference = "https://attack.mitre.org/tactics/TA0011/"
-
diff --git a/rules/network/initial_access_rdp_remote_desktop_protocol_to_the_internet.toml b/rules/network/initial_access_rdp_remote_desktop_protocol_to_the_internet.toml
index fbdb638d7..dbb30e282 100644
--- a/rules/network/initial_access_rdp_remote_desktop_protocol_to_the_internet.toml
+++ b/rules/network/initial_access_rdp_remote_desktop_protocol_to_the_internet.toml
@@ -1,8 +1,8 @@
 [metadata]
 creation_date = "2020/02/18"
-ecs_version = ["1.4.0"]
+ecs_version = ["1.5.0"]
 maturity = "production"
-updated_date = "2020/03/09"
+updated_date = "2020/07/02"
 
 [rule]
 author = ["Elastic"]
@@ -21,7 +21,7 @@ false_positives = [
     unexpected. Usage that is unfamiliar to server or network owners can be unexpected and suspicious.
     """,
 ]
-index = ["filebeat-*"]
+index = ["filebeat-*", "packetbeat-*"]
 language = "kuery"
 license = "Elastic License"
 name = "RDP (Remote Desktop Protocol) to the Internet"
@@ -32,7 +32,7 @@ tags = ["Elastic", "Network"]
 type = "query"
 
 query = '''
-network.transport:tcp and destination.port:3389 and
+event.category:(network or network_traffic) and network.transport:tcp and (destination.port:3389 or event.dataset:zeek.rdp) and
   source.ip:(10.0.0.0/8 or 172.16.0.0/12 or 192.168.0.0/16) and
   not destination.ip:(10.0.0.0/8 or 127.0.0.0/8 or 172.16.0.0/12 or 192.168.0.0/16 or "::1")
 '''
@@ -62,4 +62,3 @@ reference = "https://attack.mitre.org/techniques/T1048/"
 id = "TA0010"
 name = "Exfiltration"
 reference = "https://attack.mitre.org/tactics/TA0010/"
-
diff --git a/rules/network/initial_access_rpc_remote_procedure_call_from_the_internet.toml b/rules/network/initial_access_rpc_remote_procedure_call_from_the_internet.toml
index 63a2998e9..a0a6c27e7 100644
--- a/rules/network/initial_access_rpc_remote_procedure_call_from_the_internet.toml
+++ b/rules/network/initial_access_rpc_remote_procedure_call_from_the_internet.toml
@@ -1,8 +1,8 @@
 [metadata]
 creation_date = "2020/02/18"
-ecs_version = ["1.4.0"]
+ecs_version = ["1.5.0"]
 maturity = "production"
-updated_date = "2020/03/09"
+updated_date = "2020/07/02"
 
 [rule]
 author = ["Elastic"]
@@ -12,7 +12,7 @@ system administrators to remotely control a system for maintenance or to use sha
 directly exposed to the Internet, as it is frequently targeted and exploited by threat actors as an initial access or
 back-door vector.
 """
-index = ["filebeat-*"]
+index = ["filebeat-*", "packetbeat-*"]
 language = "kuery"
 license = "Elastic License"
 name = "RPC (Remote Procedure Call) from the Internet"
@@ -23,7 +23,7 @@ tags = ["Elastic", "Network"]
 type = "query"
 
 query = '''
-network.transport:tcp and destination.port:135 and
+event.category:(network or network_traffic) and network.transport:tcp and (destination.port:135 or event.dataset:zeek.dce_rpc) and
   not source.ip:(10.0.0.0/8 or 127.0.0.0/8 or 172.16.0.0/12 or 192.168.0.0/16 or "::1") and
   destination.ip:(10.0.0.0/8 or 172.16.0.0/12 or 192.168.0.0/16)
 '''
@@ -41,4 +41,3 @@ reference = "https://attack.mitre.org/techniques/T1190/"
 id = "TA0011"
 name = "Initial Access"
 reference = "https://attack.mitre.org/tactics/TA0011/"
-
diff --git a/rules/network/initial_access_rpc_remote_procedure_call_to_the_internet.toml b/rules/network/initial_access_rpc_remote_procedure_call_to_the_internet.toml
index e263bacaf..e17dc9174 100644
--- a/rules/network/initial_access_rpc_remote_procedure_call_to_the_internet.toml
+++ b/rules/network/initial_access_rpc_remote_procedure_call_to_the_internet.toml
@@ -1,8 +1,8 @@
 [metadata]
 creation_date = "2020/02/18"
-ecs_version = ["1.4.0"]
+ecs_version = ["1.5.0"]
 maturity = "production"
-updated_date = "2020/03/09"
+updated_date = "2020/07/02"
 
 [rule]
 author = ["Elastic"]
@@ -12,7 +12,7 @@ system administrators to remotely control a system for maintenance or to use sha
 directly exposed to the Internet, as it is frequently targeted and exploited by threat actors as an initial access or
 back-door vector.
 """
-index = ["filebeat-*"]
+index = ["filebeat-*", "packetbeat-*"]
 language = "kuery"
 license = "Elastic License"
 name = "RPC (Remote Procedure Call) to the Internet"
@@ -23,7 +23,7 @@ tags = ["Elastic", "Network"]
 type = "query"
 
 query = '''
-network.transport:tcp and destination.port:135 and
+event.category:(network or network_traffic) and network.transport:tcp and (destination.port:135 or event.dataset:zeek.dce_rpc) and
   source.ip:(10.0.0.0/8 or 172.16.0.0/12 or 192.168.0.0/16) and
   not destination.ip:(10.0.0.0/8 or 127.0.0.0/8 or 172.16.0.0/12 or 192.168.0.0/16 or "::1")
 '''
@@ -41,4 +41,3 @@ reference = "https://attack.mitre.org/techniques/T1190/"
 id = "TA0011"
 name = "Initial Access"
 reference = "https://attack.mitre.org/tactics/TA0011/"
-
diff --git a/rules/network/initial_access_smb_windows_file_sharing_activity_to_the_internet.toml b/rules/network/initial_access_smb_windows_file_sharing_activity_to_the_internet.toml
index ecf3192c7..43b30d89f 100644
--- a/rules/network/initial_access_smb_windows_file_sharing_activity_to_the_internet.toml
+++ b/rules/network/initial_access_smb_windows_file_sharing_activity_to_the_internet.toml
@@ -1,8 +1,8 @@
 [metadata]
 creation_date = "2020/02/18"
-ecs_version = ["1.4.0"]
+ecs_version = ["1.5.0"]
 maturity = "production"
-updated_date = "2020/03/09"
+updated_date = "2020/07/02"
 
 [rule]
 author = ["Elastic"]
@@ -12,7 +12,7 @@ the Internet. SMB is commonly used within networks to share files, printers, and
 systems. It should almost never be directly exposed to the Internet, as it is frequently targeted and exploited by
 threat actors as an initial access or back-door vector or for data exfiltration.
 """
-index = ["filebeat-*"]
+index = ["filebeat-*", "packetbeat-*"]
 language = "kuery"
 license = "Elastic License"
 name = "SMB (Windows File Sharing) Activity to the Internet"
@@ -23,7 +23,7 @@ tags = ["Elastic", "Network"]
 type = "query"
 
 query = '''
-network.transport:tcp and destination.port:(139 or 445) and
+event.category:(network or network_traffic) and network.transport:tcp and (destination.port:(139 or 445) or event.dataset:zeek.smb) and
   source.ip:(10.0.0.0/8 or 172.16.0.0/12 or 192.168.0.0/16) and
   not destination.ip:(10.0.0.0/8 or 127.0.0.0/8 or 172.16.0.0/12 or 192.168.0.0/16 or "::1")
 '''
@@ -53,4 +53,3 @@ reference = "https://attack.mitre.org/techniques/T1048/"
 id = "TA0010"
 name = "Exfiltration"
 reference = "https://attack.mitre.org/tactics/TA0010/"
-