Expand timestamp override tests (#1907)

* Expand timestamp_override tests
* removed timestamp_override from eql sequence rules
* add config entry for eql rules with beats index and t_o
* add timestamp_override to missing fields

rules/linux/command_and_control_tunneling_via_earthworm.toml

@@ -1,22 +1,27 @@
 [metadata]
 creation_date = "2021/04/12"
 maturity = "production"
-updated_date = "2021/04/12"
+updated_date = "2022/03/31"
 
 [rule]
 author = ["Elastic"]
 description = """
 Identifies the execution of the EarthWorm tunneler. Adversaries may tunnel network communications to and from a victim
-system within a separate protocol to avoid detection and network filtering, or to enable access to otherwise unreachable systems.
+system within a separate protocol to avoid detection and network filtering, or to enable access to otherwise unreachable
+systems.
 """
 from = "now-9m"
 index = ["auditbeat-*", "logs-endpoint.events.*"]
 language = "eql"
 license = "Elastic License v2"
 name = "Potential Protocol Tunneling via EarthWorm"
+note = """## Config
+
+If enabling an EQL rule on a non-elastic-agent index (such as beats) for versions <8.2, events will not define `event.ingested` and default fallback for EQL rules was not added until 8.2, so you will need to add a custom pipeline to populate `event.ingested` to @timestamp for this rule to work.
+"""
 references = [
     "http://rootkiter.com/EarthWorm/",
-    "https://decoded.avast.io/luigicamastra/apt-group-targeting-governmental-agencies-in-east-asia/"
+    "https://decoded.avast.io/luigicamastra/apt-group-targeting-governmental-agencies-in-east-asia/",
 ]
 risk_score = 47
 rule_id = "9f1c4ca3-44b5-481d-ba42-32dc215a2769"
@@ -38,7 +43,9 @@ id = "T1572"
 name = "Protocol Tunneling"
 reference = "https://attack.mitre.org/techniques/T1572/"
 
+
 [rule.threat.tactic]
 id = "TA0011"
 name = "Command and Control"
 reference = "https://attack.mitre.org/tactics/TA0011/"
+

rules/linux/credential_access_ssh_backdoor_log.toml

@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2020/12/21"
 maturity = "production"
-updated_date = "2021/03/03"
+updated_date = "2022/03/31"
 
 [rule]
 author = ["Elastic"]
@@ -16,6 +16,10 @@ index = ["auditbeat-*", "logs-endpoint.events.*"]
 language = "eql"
 license = "Elastic License v2"
 name = "Potential OpenSSH Backdoor Logging Activity"
+note = """## Config
+
+If enabling an EQL rule on a non-elastic-agent index (such as beats) for versions <8.2, events will not define `event.ingested` and default fallback for EQL rules was not added until 8.2, so you will need to add a custom pipeline to populate `event.ingested` to @timestamp for this rule to work.
+"""
 references = [
     "https://github.com/eset/malware-ioc/tree/master/sshdoor",
     "https://www.welivesecurity.com/wp-content/uploads/2021/01/ESET_Kobalos.pdf",

rules/linux/defense_evasion_deletion_of_bash_command_line_history.toml

@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2020/05/04"
 maturity = "production"
-updated_date = "2021/03/03"
+updated_date = "2022/03/31"
 
 [rule]
 author = ["Elastic"]
@@ -14,6 +14,10 @@ index = ["auditbeat-*", "logs-endpoint.events.*"]
 language = "eql"
 license = "Elastic License v2"
 name = "Tampering of Bash Command-Line History"
+note = """## Config
+
+If enabling an EQL rule on a non-elastic-agent index (such as beats) for versions <8.2, events will not define `event.ingested` and default fallback for EQL rules was not added until 8.2, so you will need to add a custom pipeline to populate `event.ingested` to @timestamp for this rule to work.
+"""
 risk_score = 47
 rule_id = "7bcbb3ac-e533-41ad-a612-d6c3bf666aba"
 severity = "medium"

rules/linux/defense_evasion_hidden_file_dir_tmp.toml

@@ -1,9 +1,9 @@
 [metadata]
 creation_date = "2020/04/29"
 maturity = "production"
-updated_date = "2021/03/03"
 min_stack_comments = "EQL regex syntax introduced in 7.12"
 min_stack_version = "7.12.0"
+updated_date = "2022/03/31"
 
 [rule]
 author = ["Elastic"]
@@ -24,6 +24,10 @@ language = "eql"
 license = "Elastic License v2"
 max_signals = 33
 name = "Creation of Hidden Files and Directories"
+note = """## Config
+
+If enabling an EQL rule on a non-elastic-agent index (such as beats) for versions <8.2, events will not define `event.ingested` and default fallback for EQL rules was not added until 8.2, so you will need to add a custom pipeline to populate `event.ingested` to @timestamp for this rule to work.
+"""
 risk_score = 47
 rule_id = "b9666521-4742-49ce-9ddc-b8e84c35acae"
 severity = "medium"

rules/linux/defense_evasion_log_files_deleted.toml

@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2020/11/03"
 maturity = "production"
-updated_date = "2021/03/03"
+updated_date = "2022/03/31"
 
 [rule]
 author = ["Elastic"]
@@ -14,6 +14,10 @@ index = ["auditbeat-*", "logs-endpoint.events.*"]
 language = "eql"
 license = "Elastic License v2"
 name = "System Log File Deletion"
+note = """## Config
+
+If enabling an EQL rule on a non-elastic-agent index (such as beats) for versions <8.2, events will not define `event.ingested` and default fallback for EQL rules was not added until 8.2, so you will need to add a custom pipeline to populate `event.ingested` to @timestamp for this rule to work.
+"""
 references = [
     "https://www.fireeye.com/blog/threat-research/2020/11/live-off-the-land-an-overview-of-unc1945.html",
 ]

rules/linux/execution_apt_binary.toml

@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2022/02/24"
 maturity = "production"
-updated_date = "2022/03/28"
+updated_date = "2022/03/31"
 
 [rule]
 author = ["Elastic"]
@@ -26,8 +26,8 @@ timestamp_override = "event.ingested"
 type = "eql"
 
 query = '''
-process where event.type == "start" and process.name == "sensible-pager" and 
-  process.args in ("/bin/sh", "/bin/bash", "/bin/dash", "sh", "bash", "dash") and 
+process where event.type == "start" and process.name == "sensible-pager" and
+  process.args in ("/bin/sh", "/bin/bash", "/bin/dash", "sh", "bash", "dash") and
   process.parent.name in ("apt", "apt-get") and process.parent.args == "changelog"
 '''
 

rules/linux/execution_awk_binary_shell.toml

@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2022/02/24"
 maturity = "production"
-updated_date = "2022/03/28"
+updated_date = "2022/03/31"
 
 [rule]
 author = ["Elastic"]
@@ -25,7 +25,7 @@ timestamp_override = "event.ingested"
 type = "eql"
 
 query = '''
-process where event.type == "start" and process.name in ("sh", "bash", "dash") and 
+process where event.type == "start" and process.name in ("sh", "bash", "dash") and
   process.parent.name in ("nawk", "mawk", "awk", "gawk") and process.parent.args : "BEGIN {system(*)}"
 '''
 

rules/linux/execution_c89_c99_binary.toml

@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2022/03/15"
 maturity = "production"
-updated_date = "2022/03/24"
+updated_date = "2022/03/31"
 
 [rule]
 author = ["Elastic"]
@@ -48,3 +48,4 @@ reference = "https://attack.mitre.org/techniques/T1059/004/"
 id = "TA0002"
 name = "Execution"
 reference = "https://attack.mitre.org/tactics/TA0002/"
+

rules/linux/execution_cpulimit_binary.toml

@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2022/03/17"
 maturity = "production"
-updated_date = "2022/03/24"
+updated_date = "2022/03/31"
 
 [rule]
 author = ["Elastic"]
@@ -26,9 +26,9 @@ timestamp_override = "event.ingested"
 type = "eql"
 
 query = '''
-process where event.type == "start" and process.name in ("bash", "sh", "dash") and 
-  process.parent.name == "cpulimit" and process.parent.args == "-f" and 
-  process.parent.args in ("/bin/sh", "/bin/bash", "/bin/dash", "sh", "bash", "dash") 
+process where event.type == "start" and process.name in ("bash", "sh", "dash") and
+  process.parent.name == "cpulimit" and process.parent.args == "-f" and
+  process.parent.args in ("/bin/sh", "/bin/bash", "/bin/dash", "sh", "bash", "dash")
 '''
 
 

rules/linux/execution_crash_binary.toml

@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2022/03/21"
 maturity = "production"
-updated_date = "2022/03/24"
+updated_date = "2022/03/31"
 
 [rule]
 author = ["Elastic"]

rules/linux/execution_expect_binary.toml

@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2022/03/07"
 maturity = "production"
-updated_date = "2022/03/28"
+updated_date = "2022/03/31"
 
 [rule]
 author = ["Elastic"]
@@ -25,9 +25,9 @@ timestamp_override = "event.ingested"
 type = "eql"
 
 query = '''
-process where event.type == "start" and process.name in ("bash", "sh", "dash") and 
-  process.parent.name == "expect" and process.parent.args == "-c" and 
-  process.parent.args in ("spawn /bin/sh;interact", "spawn /bin/bash;interact", "spawn /bin/dash;interact", "spawn sh;interact", "spawn bash;interact", "spawn dash;interact") 
+process where event.type == "start" and process.name in ("bash", "sh", "dash") and
+  process.parent.name == "expect" and process.parent.args == "-c" and
+  process.parent.args in ("spawn /bin/sh;interact", "spawn /bin/bash;interact", "spawn /bin/dash;interact", "spawn sh;interact", "spawn bash;interact", "spawn dash;interact")
 '''
 
 

rules/linux/execution_find_binary.toml

@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2022/02/28"
 maturity = "production"
-updated_date = "2022/03/28"
+updated_date = "2022/03/31"
 
 [rule]
 author = ["Elastic"]
@@ -25,8 +25,8 @@ timestamp_override = "event.ingested"
 type = "eql"
 
 query = '''
-process where event.type == "start" and process.name in ("bash", "sh") and 
-  process.parent.name == "find" and process.parent.args == "-exec" and 
+process where event.type == "start" and process.name in ("bash", "sh") and
+  process.parent.name == "find" and process.parent.args == "-exec" and
   process.parent.args == ";" and process.parent.args in ("/bin/bash", "/bin/sh", "bash", "sh")
 '''
 

rules/linux/execution_gcc_binary.toml

@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2022/03/09"
 maturity = "production"
-updated_date = "2022/03/28"
+updated_date = "2022/03/31"
 
 [rule]
 author = ["Elastic"]
@@ -25,8 +25,8 @@ timestamp_override = "event.ingested"
 type = "eql"
 
 query = '''
-process where event.type == "start" and process.name in ("sh", "dash", "bash")  and 
-  process.parent.name == "gcc" and process.parent.args == "-wrapper" and 
+process where event.type == "start" and process.name in ("sh", "dash", "bash")  and
+  process.parent.name == "gcc" and process.parent.args == "-wrapper" and
   process.parent.args in ("sh,-s", "bash,-s", "dash,-s", "/bin/sh,-s", "/bin/bash,-s", "/bin/dash,-s")
 '''
 

rules/linux/execution_mysql_binary.toml

@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2022/03/09"
 maturity = "production"
-updated_date = "2022/03/28"
+updated_date = "2022/03/31"
 
 [rule]
 author = ["Elastic"]
@@ -25,9 +25,9 @@ timestamp_override = "event.ingested"
 type = "eql"
 
 query = '''
-process where event.type == "start" and process.name in ("bash", "sh", "dash") and 
-  process.parent.name == "mysql" and process.parent.args == "-e" and 
-  process.parent.args : ("\\!*sh", "\\!*bash", "\\!*dash", "\\!*/bin/sh", "\\!*/bin/bash", "\\!*/bin/dash") 
+process where event.type == "start" and process.name in ("bash", "sh", "dash") and
+  process.parent.name == "mysql" and process.parent.args == "-e" and
+  process.parent.args : ("\\!*sh", "\\!*bash", "\\!*dash", "\\!*/bin/sh", "\\!*/bin/bash", "\\!*/bin/dash")
 '''
 
 

rules/linux/execution_nice_binary.toml

@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2022/03/07"
 maturity = "development"
-updated_date = "2022/03/28"
+updated_date = "2022/03/31"
 
 [rule]
 author = ["Elastic"]
@@ -26,7 +26,7 @@ timestamp_override = "event.ingested"
 type = "eql"
 
 query = '''
-process where event.type == "start" and process.name in ("bash", "sh", "dash") and 
+process where event.type == "start" and process.name in ("bash", "sh", "dash") and
   process.parent.name == "nice" and process.parent.args in ("/bin/bash", "/bin/sh", "/bin/dash", "sh", "bash", "dash")
 '''
 

rules/linux/execution_ssh_binary.toml

@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2022/03/10"
 maturity = "production"
-updated_date = "2022/03/28"
+updated_date = "2022/03/31"
 
 [rule]
 author = ["Elastic"]
@@ -25,8 +25,8 @@ timestamp_override = "event.ingested"
 type = "eql"
 
 query = '''
-process where event.type == "start" and process.name : ("bash", "sh", "dash") and 
-  process.parent.name == "ssh" and process.parent.args == "-o" and 
+process where event.type == "start" and process.name : ("bash", "sh", "dash") and
+  process.parent.name == "ssh" and process.parent.args == "-o" and
   process.parent.args in ("ProxyCommand=;sh 0<&2 1>&2", "ProxyCommand=;bash 0<&2 1>&2", "ProxyCommand=;dash 0<&2 1>&2", "ProxyCommand=;/bin/sh 0<&2 1>&2", "ProxyCommand=;/bin/bash 0<&2 1>&2", "ProxyCommand=;/bin/dash 0<&2 1>&2")
 '''
 

rules/linux/persistence_kde_autostart_modification.toml

@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2021/01/06"
 maturity = "production"
-updated_date = "2021/03/03"
+updated_date = "2022/03/31"
 
 [rule]
 author = ["Elastic"]
@@ -14,6 +14,10 @@ index = ["auditbeat-*", "logs-endpoint.events.*"]
 language = "eql"
 license = "Elastic License v2"
 name = "Persistence via KDE AutoStart Script or Desktop File Modification"
+note = """## Config
+
+If enabling an EQL rule on a non-elastic-agent index (such as beats) for versions <8.2, events will not define `event.ingested` and default fallback for EQL rules was not added until 8.2, so you will need to add a custom pipeline to populate `event.ingested` to @timestamp for this rule to work.
+"""
 references = [
     "https://userbase.kde.org/System_Settings/Autostart",
     "https://www.amnesty.org/en/latest/research/2020/09/german-made-finspy-spyware-found-in-egypt-and-mac-and-linux-versions-revealed/",