Expand timestamp override tests (#1907)

* Expand timestamp_override tests
* removed timestamp_override from eql sequence rules
* add config entry for eql rules with beats index and t_o
* add timestamp_override to missing fields

rules/macos/credential_access_access_to_browser_credentials_procargs.toml

@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2020/01/04"
 maturity = "production"
-updated_date = "2021/03/03"
+updated_date = "2022/03/31"
 
 [rule]
 author = ["Elastic"]
@@ -14,6 +14,10 @@ index = ["auditbeat-*", "logs-endpoint.events.*"]
 language = "eql"
 license = "Elastic License v2"
 name = "Access of Stored Browser Credentials"
+note = """## Config
+
+If enabling an EQL rule on a non-elastic-agent index (such as beats) for versions <8.2, events will not define `event.ingested` and default fallback for EQL rules was not added until 8.2, so you will need to add a custom pipeline to populate `event.ingested` to @timestamp for this rule to work.
+"""
 references = ["https://securelist.com/calisto-trojan-for-macos/86543/"]
 risk_score = 73
 rule_id = "20457e4f-d1de-4b92-ae69-142e27a4342a"

rules/macos/credential_access_credentials_keychains.toml

@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2020/08/14"
 maturity = "production"
-updated_date = "2021/03/08"
+updated_date = "2022/03/31"
 
 [rule]
 author = ["Elastic"]
@@ -15,6 +15,10 @@ index = ["auditbeat-*", "logs-endpoint.events.*"]
 language = "eql"
 license = "Elastic License v2"
 name = "Access to Keychain Credentials Directories"
+note = """## Config
+
+If enabling an EQL rule on a non-elastic-agent index (such as beats) for versions <8.2, events will not define `event.ingested` and default fallback for EQL rules was not added until 8.2, so you will need to add a custom pipeline to populate `event.ingested` to @timestamp for this rule to work.
+"""
 references = [
     "https://objective-see.com/blog/blog_0x25.html",
     "https://securelist.com/calisto-trojan-for-macos/86543/",
@@ -68,3 +72,4 @@ reference = "https://attack.mitre.org/techniques/T1555/001/"
 id = "TA0006"
 name = "Credential Access"
 reference = "https://attack.mitre.org/tactics/TA0006/"
+

rules/macos/credential_access_dumping_keychain_security.toml

@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2021/01/04"
 maturity = "production"
-updated_date = "2021/03/03"
+updated_date = "2022/03/31"
 
 [rule]
 author = ["Elastic"]
@@ -15,6 +15,10 @@ index = ["auditbeat-*", "logs-endpoint.events.*"]
 language = "eql"
 license = "Elastic License v2"
 name = "Dumping of Keychain Content via Security Command"
+note = """## Config
+
+If enabling an EQL rule on a non-elastic-agent index (such as beats) for versions <8.2, events will not define `event.ingested` and default fallback for EQL rules was not added until 8.2, so you will need to add a custom pipeline to populate `event.ingested` to @timestamp for this rule to work.
+"""
 references = ["https://ss64.com/osx/security.html"]
 risk_score = 73
 rule_id = "565d6ca5-75ba-4c82-9b13-add25353471c"

rules/macos/credential_access_keychain_pwd_retrieval_security_cmd.toml

@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2020/01/06"
 maturity = "production"
-updated_date = "2021/03/08"
+updated_date = "2022/03/31"
 
 [rule]
 author = ["Elastic"]
@@ -16,6 +16,10 @@ index = ["auditbeat-*", "logs-endpoint.events.*"]
 language = "eql"
 license = "Elastic License v2"
 name = "Keychain Password Retrieval via Command Line"
+note = """## Config
+
+If enabling an EQL rule on a non-elastic-agent index (such as beats) for versions <8.2, events will not define `event.ingested` and default fallback for EQL rules was not added until 8.2, so you will need to add a custom pipeline to populate `event.ingested` to @timestamp for this rule to work.
+"""
 references = [
     "https://www.netmeister.org/blog/keychain-passwords.html",
     "https://github.com/priyankchheda/chrome_password_grabber/blob/master/chrome.py",
@@ -48,6 +52,7 @@ id = "T1555.001"
 name = "Keychain"
 reference = "https://attack.mitre.org/techniques/T1555/001/"
 
+
 [[rule.threat.technique]]
 id = "T1555"
 name = "Credentials from Password Stores"
@@ -58,7 +63,9 @@ name = "Credentials from Web Browsers"
 reference = "https://attack.mitre.org/techniques/T1555/003/"
 
 
+
 [rule.threat.tactic]
 id = "TA0006"
 name = "Credential Access"
 reference = "https://attack.mitre.org/tactics/TA0006/"
+

rules/macos/credential_access_promt_for_pwd_via_osascript.toml

@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2020/11/16"
 maturity = "production"
-updated_date = "2021/03/03"
+updated_date = "2022/03/31"
 
 [rule]
 author = ["Elastic"]
@@ -14,6 +14,10 @@ index = ["auditbeat-*", "logs-endpoint.events.*"]
 language = "eql"
 license = "Elastic License v2"
 name = "Prompt for Credentials with OSASCRIPT"
+note = """## Config
+
+If enabling an EQL rule on a non-elastic-agent index (such as beats) for versions <8.2, events will not define `event.ingested` and default fallback for EQL rules was not added until 8.2, so you will need to add a custom pipeline to populate `event.ingested` to @timestamp for this rule to work.
+"""
 references = [
     "https://github.com/EmpireProject/EmPyre/blob/master/lib/modules/collection/osx/prompt.py",
     "https://ss64.com/osx/osascript.html",

rules/macos/defense_evasion_attempt_del_quarantine_attrib.toml

@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2020/08/14"
 maturity = "production"
-updated_date = "2021/03/03"
+updated_date = "2022/03/31"
 
 [rule]
 author = ["Elastic"]
@@ -15,6 +15,10 @@ index = ["auditbeat-*", "logs-endpoint.events.*"]
 language = "eql"
 license = "Elastic License v2"
 name = "Attempt to Remove File Quarantine Attribute"
+note = """## Config
+
+If enabling an EQL rule on a non-elastic-agent index (such as beats) for versions <8.2, events will not define `event.ingested` and default fallback for EQL rules was not added until 8.2, so you will need to add a custom pipeline to populate `event.ingested` to @timestamp for this rule to work.
+"""
 references = [
     "https://www.trendmicro.com/en_us/research/20/k/new-macos-backdoor-connected-to-oceanlotus-surfaces.html",
     "https://ss64.com/osx/xattr.html",

rules/macos/defense_evasion_privacy_controls_tcc_database_modification.toml

@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2020/12/23"
 maturity = "production"
-updated_date = "2021/08/25"
+updated_date = "2022/03/31"
 
 [rule]
 author = ["Elastic"]
@@ -15,6 +15,10 @@ index = ["auditbeat-*", "logs-endpoint.events.*"]
 language = "eql"
 license = "Elastic License v2"
 name = "Potential Privacy Control Bypass via TCCDB Modification"
+note = """## Config
+
+If enabling an EQL rule on a non-elastic-agent index (such as beats) for versions <8.2, events will not define `event.ingested` and default fallback for EQL rules was not added until 8.2, so you will need to add a custom pipeline to populate `event.ingested` to @timestamp for this rule to work.
+"""
 references = [
     "https://applehelpwriter.com/2016/08/29/discovering-how-dropbox-hacks-your-mac/",
     "https://github.com/bp88/JSS-Scripts/blob/master/TCC.db%20Modifier.sh",

rules/macos/defense_evasion_privilege_escalation_privacy_pref_sshd_fulldiskaccess.toml

@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2020/01/11"
 maturity = "production"
-updated_date = "2021/03/03"
+updated_date = "2022/03/31"
 
 [rule]
 author = ["Elastic"]
@@ -15,6 +15,10 @@ index = ["auditbeat-*", "logs-endpoint.events.*"]
 language = "eql"
 license = "Elastic License v2"
 name = "Potential Privacy Control Bypass via Localhost Secure Copy"
+note = """## Config
+
+If enabling an EQL rule on a non-elastic-agent index (such as beats) for versions <8.2, events will not define `event.ingested` and default fallback for EQL rules was not added until 8.2, so you will need to add a custom pipeline to populate `event.ingested` to @timestamp for this rule to work.
+"""
 references = [
     "https://blog.trendmicro.com/trendlabs-security-intelligence/xcsset-mac-malware-infects-xcode-projects-performs-uxss-attack-on-safari-other-browsers-leverages-zero-day-exploits/",
 ]

rules/macos/discovery_users_domain_built_in_commands.toml

@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2021/01/12"
 maturity = "production"
-updated_date = "2022/03/28"
+updated_date = "2022/03/31"
 
 [rule]
 author = ["Elastic"]
@@ -11,6 +11,10 @@ index = ["auditbeat-*", "logs-endpoint.events.*"]
 language = "eql"
 license = "Elastic License v2"
 name = "Enumeration of Users or Groups via Built-in Commands"
+note = """## Config
+
+If enabling an EQL rule on a non-elastic-agent index (such as beats) for versions <8.2, events will not define `event.ingested` and default fallback for EQL rules was not added until 8.2, so you will need to add a custom pipeline to populate `event.ingested` to @timestamp for this rule to work.
+"""
 risk_score = 21
 rule_id = "6e9b351e-a531-4bdc-b73e-7034d6eed7ff"
 severity = "low"
@@ -51,3 +55,4 @@ reference = "https://attack.mitre.org/techniques/T1087/"
 id = "TA0007"
 name = "Discovery"
 reference = "https://attack.mitre.org/tactics/TA0007/"
+

rules/macos/execution_installer_spawned_network_event.toml

@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2021/02/23"
 maturity = "production"
-updated_date = "2021/08/03"
+updated_date = "2022/03/31"
 
 [rule]
 author = ["Elastic"]
@@ -28,7 +28,6 @@ risk_score = 47
 rule_id = "99239e7d-b0d4-46e3-8609-acafcf99f68c"
 severity = "medium"
 tags = ["Elastic", "Host", "macOS", "Threat Detection", "Execution"]
-timestamp_override = "event.ingested"
 type = "eql"
 
 query = '''

rules/macos/lateral_movement_mounting_smb_share.toml

@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2021/01/25"
 maturity = "production"
-updated_date = "2021/03/03"
+updated_date = "2022/03/31"
 
 [rule]
 author = ["Elastic"]
@@ -14,6 +14,10 @@ index = ["auditbeat-*", "logs-endpoint.events.*"]
 language = "eql"
 license = "Elastic License v2"
 name = "Attempt to Mount SMB Share via Command Line"
+note = """## Config
+
+If enabling an EQL rule on a non-elastic-agent index (such as beats) for versions <8.2, events will not define `event.ingested` and default fallback for EQL rules was not added until 8.2, so you will need to add a custom pipeline to populate `event.ingested` to @timestamp for this rule to work.
+"""
 references = ["https://www.freebsd.org/cgi/man.cgi?mount_smbfs", "https://ss64.com/osx/mount.html"]
 risk_score = 21
 rule_id = "661545b4-1a90-4f45-85ce-2ebd7c6a15d0"

rules/macos/lateral_movement_vpn_connection_attempt.toml

@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2020/01/25"
 maturity = "production"
-updated_date = "2021/03/03"
+updated_date = "2022/03/31"
 
 [rule]
 author = ["Elastic"]
@@ -11,6 +11,10 @@ index = ["auditbeat-*", "logs-endpoint.events.*"]
 language = "eql"
 license = "Elastic License v2"
 name = "Virtual Private Network Connection Attempt"
+note = """## Config
+
+If enabling an EQL rule on a non-elastic-agent index (such as beats) for versions <8.2, events will not define `event.ingested` and default fallback for EQL rules was not added until 8.2, so you will need to add a custom pipeline to populate `event.ingested` to @timestamp for this rule to work.
+"""
 references = [
     "https://github.com/rapid7/metasploit-framework/blob/master/modules/post/osx/manage/vpn.rb",
     "https://www.unix.com/man-page/osx/8/networksetup/",

rules/macos/persistence_creation_hidden_login_item_osascript.toml

@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2020/01/05"
 maturity = "production"
-updated_date = "2021/03/03"
+updated_date = "2022/03/31"
 
 [rule]
 author = ["Elastic"]
@@ -14,6 +14,10 @@ index = ["auditbeat-*", "logs-endpoint.events.*"]
 language = "eql"
 license = "Elastic License v2"
 name = "Creation of Hidden Login Item via Apple Script"
+note = """## Config
+
+If enabling an EQL rule on a non-elastic-agent index (such as beats) for versions <8.2, events will not define `event.ingested` and default fallback for EQL rules was not added until 8.2, so you will need to add a custom pipeline to populate `event.ingested` to @timestamp for this rule to work.
+"""
 risk_score = 47
 rule_id = "f24bcae1-8980-4b30-b5dd-f851b055c9e7"
 severity = "medium"

rules/macos/persistence_emond_rules_file_creation.toml

@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2021/01/11"
 maturity = "production"
-updated_date = "2021/03/03"
+updated_date = "2022/03/31"
 
 [rule]
 author = ["Elastic"]
@@ -14,6 +14,10 @@ index = ["auditbeat-*", "logs-endpoint.events.*"]
 language = "eql"
 license = "Elastic License v2"
 name = "Emond Rules Creation or Modification"
+note = """## Config
+
+If enabling an EQL rule on a non-elastic-agent index (such as beats) for versions <8.2, events will not define `event.ingested` and default fallback for EQL rules was not added until 8.2, so you will need to add a custom pipeline to populate `event.ingested` to @timestamp for this rule to work.
+"""
 references = ["https://www.xorrior.com/emond-persistence/"]
 risk_score = 47
 rule_id = "a6bf4dd4-743e-4da8-8c03-3ebd753a6c90"

rules/macos/persistence_evasion_hidden_launch_agent_deamon_creation.toml

@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2020/01/05"
 maturity = "production"
-updated_date = "2021/03/03"
+updated_date = "2022/03/31"
 
 [rule]
 author = ["Elastic"]
@@ -14,6 +14,10 @@ index = ["auditbeat-*", "logs-endpoint.events.*"]
 language = "eql"
 license = "Elastic License v2"
 name = "Creation of Hidden Launch Agent or Daemon"
+note = """## Config
+
+If enabling an EQL rule on a non-elastic-agent index (such as beats) for versions <8.2, events will not define `event.ingested` and default fallback for EQL rules was not added until 8.2, so you will need to add a custom pipeline to populate `event.ingested` to @timestamp for this rule to work.
+"""
 references = [
     "https://developer.apple.com/library/archive/documentation/MacOSX/Conceptual/BPSystemStartup/Chapters/CreatingLaunchdJobs.html",
 ]

rules/macos/persistence_login_logout_hooks_defaults.toml

@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2020/12/07"
 maturity = "production"
-updated_date = "2021/03/09"
+updated_date = "2022/03/31"
 
 [rule]
 author = ["Elastic"]
@@ -14,6 +14,10 @@ index = ["auditbeat-*", "logs-endpoint.events.*"]
 language = "eql"
 license = "Elastic License v2"
 name = "Persistence via Login or Logout Hook"
+note = """## Config
+
+If enabling an EQL rule on a non-elastic-agent index (such as beats) for versions <8.2, events will not define `event.ingested` and default fallback for EQL rules was not added until 8.2, so you will need to add a custom pipeline to populate `event.ingested` to @timestamp for this rule to work.
+"""
 references = [
     "https://www.virusbulletin.com/uploads/pdf/conference_slides/2014/Wardle-VB2014.pdf",
     "https://www.manpagez.com/man/1/defaults/",
@@ -50,3 +54,4 @@ reference = "https://attack.mitre.org/techniques/T1037/"
 id = "TA0003"
 name = "Persistence"
 reference = "https://attack.mitre.org/tactics/TA0003/"
+

rules/macos/persistence_modification_sublime_app_plugin_or_script.toml

@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2020/12/23"
 maturity = "production"
-updated_date = "2021/03/03"
+updated_date = "2022/03/31"
 
 [rule]
 author = ["Elastic"]
@@ -14,6 +14,10 @@ index = ["auditbeat-*", "logs-endpoint.events.*"]
 language = "eql"
 license = "Elastic License v2"
 name = "Sublime Plugin or Application Script Modification"
+note = """## Config
+
+If enabling an EQL rule on a non-elastic-agent index (such as beats) for versions <8.2, events will not define `event.ingested` and default fallback for EQL rules was not added until 8.2, so you will need to add a custom pipeline to populate `event.ingested` to @timestamp for this rule to work.
+"""
 references = ["https://posts.specterops.io/persistent-jxa-66e1c3cd1cf5"]
 risk_score = 21
 rule_id = "88817a33-60d3-411f-ba79-7c905d865b2a"

rules/macos/persistence_screensaver_engine_unexpected_child_process.toml

@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2021/10/05"
 maturity = "production"
-updated_date = "2021/10/05"
+updated_date = "2022/03/31"
 
 [rule]
 author = ["Elastic"]
@@ -22,6 +22,11 @@ note = """## Triage and analysis
 as a download of a payload from a server.
 - Review the installed and activated screensaver on the host. Triage the screensaver (.saver) file that was triggered to
 identify whether the file is malicious or not.
+
+
+## Config
+
+If enabling an EQL rule on a non-elastic-agent index (such as beats) for versions <8.2, events will not define `event.ingested` and default fallback for EQL rules was not added until 8.2, so you will need to add a custom pipeline to populate `event.ingested` to @timestamp for this rule to work.
 """
 references = [
     "https://posts.specterops.io/saving-your-access-d562bf5bf90b",
@@ -42,9 +47,9 @@ process where event.type == "start" and process.parent.name == "ScreenSaverEngin
 [[rule.threat]]
 framework = "MITRE ATT&CK"
 [[rule.threat.technique]]
-reference = "https://attack.mitre.org/techniques/T1546/"
-name = "Event Triggered Execution"
 id = "T1546"
+name = "Event Triggered Execution"
+reference = "https://attack.mitre.org/techniques/T1546/"
 
 
 [rule.threat.tactic]

rules/macos/persistence_screensaver_plist_file_modification.toml

@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2021/10/05"
 maturity = "production"
-updated_date = "2021/10/05"
+updated_date = "2022/03/31"
 
 [rule]
 author = ["Elastic"]
@@ -19,7 +19,12 @@ note = """## Triage and analysis
 
 - Analyze the plist file modification event to identify whether the change was expected or not
 - Investigate the process that modified the plist file for malicious code or other suspicious behavior
-- Identify if any suspicious or known malicious screensaver (.saver) files were recently written to or modified on the host"""
+- Identify if any suspicious or known malicious screensaver (.saver) files were recently written to or modified on the host
+
+## Config
+
+If enabling an EQL rule on a non-elastic-agent index (such as beats) for versions <8.2, events will not define `event.ingested` and default fallback for EQL rules was not added until 8.2, so you will need to add a custom pipeline to populate `event.ingested` to @timestamp for this rule to work.
+"""
 references = [
     "https://posts.specterops.io/saving-your-access-d562bf5bf90b",
     "https://github.com/D00MFist/PersistentJXA",
@@ -52,9 +57,9 @@ file where event.type != "deletion" and
 [[rule.threat]]
 framework = "MITRE ATT&CK"
 [[rule.threat.technique]]
-reference = "https://attack.mitre.org/techniques/T1546/"
-name = "Event Triggered Execution"
 id = "T1546"
+name = "Event Triggered Execution"
+reference = "https://attack.mitre.org/techniques/T1546/"
 
 
 [rule.threat.tactic]

rules/macos/privilege_escalation_applescript_with_admin_privs.toml

@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2020/12/27"
 maturity = "production"
-updated_date = "2021/03/03"
+updated_date = "2022/03/31"
 
 [rule]
 author = ["Elastic"]
@@ -14,6 +14,10 @@ index = ["auditbeat-*", "logs-endpoint.events.*"]
 language = "eql"
 license = "Elastic License v2"
 name = "Apple Scripting Execution with Administrator Privileges"
+note = """## Config
+
+If enabling an EQL rule on a non-elastic-agent index (such as beats) for versions <8.2, events will not define `event.ingested` and default fallback for EQL rules was not added until 8.2, so you will need to add a custom pipeline to populate `event.ingested` to @timestamp for this rule to work.
+"""
 references = ["https://discussions.apple.com/thread/2266150"]
 risk_score = 47
 rule_id = "827f8d8f-4117-4ae4-b551-f56d54b9da6b"