[BUG] test_all_rule_queries_optimized does not run on rules (#2823)

* Fixed kql -> kuery in test_all_rule_queries_opt... * all queries optimized * manually reconciled all rules that failed due to toml escaped chars * merge rules from main * Rules needing optimization * Fix optimized note * fix another note * another note fix * fixing whitespace * Updated for readability --------- Co-authored-by: terrancedejesus <terrance.dejesus@elastic.co> Co-authored-by: Mika Ayenson <Mikaayenson@users.noreply.github.com>
2023-06-23 14:58:31 +00:00
parent d829b145ef
commit aaa4ce2ea0
14 changed files with 201 additions and 190 deletions
@@ -4,15 +4,15 @@ integration = ["endpoint"]
 maturity = "production"
 min_stack_comments = "Multiple field support in the New Terms rule type was added in Elastic 8.6"
 min_stack_version = "8.6.0"
-updated_date = "2023/06/14"
+updated_date = "2023/06/22"

 [rule]
 author = ["Elastic"]
 description = """
 This rule monitors for network connectivity to the internet from a previously unknown executable located in a suspicious
-directory to a previously unknown destination ip. An alert from this rule can indicate the presence of potentially 
-malicious activity, such as the execution of unauthorized or suspicious processes attempting to establish connections to 
-unknown or suspicious destinations such as a command and control server. Detecting and investigating such behavior can 
+directory to a previously unknown destination ip. An alert from this rule can indicate the presence of potentially
+malicious activity, such as the execution of unauthorized or suspicious processes attempting to establish connections to
+unknown or suspicious destinations such as a command and control server. Detecting and investigating such behavior can
 help identify and mitigate potential security threats, protecting the system and its data from potential compromise.
 """
 from = "now-59m"
@@ -26,64 +26,77 @@ severity = "low"
 tags = ["Domain: Endpoint", "OS: Linux", "Use Case: Threat Detection", "Tactic: Command and Control", "Data Source: Elastic Endgame"]
 timestamp_override = "event.ingested"
 type = "new_terms"
+
 query = '''
-host.os.type : "linux" and event.category : "network" and 
-event.action : ("connection_attempted" or "ipv4_connection_attempt_event") and 
+host.os.type:linux and event.category:network and 
+event.action:(connection_attempted or ipv4_connection_attempt_event) and 
 process.executable : ( 
-  /tmp/* or /var/tmp/* or /dev/shm/* or /etc/init.d/* or /etc/rc*.d/* or /etc/crontab or /etc/cron.*/* or 
-  /etc/update-motd.d/* or /usr/lib/update-notifier/* or /home/*/.* or /boot/* or /srv/* or /run/* or /etc/rc.local) and
-source.ip : (
-  127.0.0.0/8 or
-  10.0.0.0/8 or
-  172.16.0.0/12 or
-  192.168.0.0/16 
-  ) and not destination.ip : (
-  10.0.0.0/8 or
-  127.0.0.0/8 or
-  169.254.0.0/16 or
-  172.16.0.0/12 or
-  192.0.0.0/24 or
-  192.0.0.0/29 or
-  192.0.0.8/32 or
-  192.0.0.9/32 or
-  192.0.0.10/32 or
-  192.0.0.170/32 or
-  192.0.0.171/32 or
-  192.0.2.0/24 or
-  192.31.196.0/24 or
-  192.52.193.0/24 or
-  192.168.0.0/16 or
-  192.88.99.0/24 or
-  224.0.0.0/4 or
-  100.64.0.0/10 or
-  192.175.48.0/24 or
-  198.18.0.0/15 or
-  198.51.100.0/24 or
-  203.0.113.0/24 or
-  240.0.0.0/4 or
-  "::1" or
-  "FE80::/10" or
-  "FF00::/8"
-  ) and not process.executable : (
-  "/usr/bin/wget" or
-  "/usr/bin/curl" or 
-  "/usr/bin/apt" or 
-  "/usr/bin/dpkg" or 
-  "/usr/bin/yum" or
-  "/usr/bin/rpm" or
-  "/usr/bin/dnf" or
-  "/usr/bin/dockerd"
-  )
+    (/etc/crontab or 
+     /etc/rc.local or 
+     /boot/* or 
+     /dev/shm/* or 
+     /etc/cron.*/* or 
+     /etc/init.d/* or 
+     /etc/rc*.d/* or 
+     /etc/update-motd.d/* or 
+     /home/*/.* or 
+     /run/* or 
+     /srv/* or 
+     /tmp/* or 
+     /usr/lib/update-notifier/* or 
+     /var/tmp/*) and 
+     not (/usr/bin/apt or 
+          /usr/bin/curl or 
+          /usr/bin/dnf or 
+          /usr/bin/dockerd or 
+          /usr/bin/dpkg or 
+          /usr/bin/rpm or 
+          /usr/bin/wget or 
+          /usr/bin/yum) 
+    ) 
+and source.ip : ( 
+    10.0.0.0/8 or 
+    127.0.0.0/8 or 
+    172.16.0.0/12 or 
+    192.168.0.0/16) and 
+    not destination.ip : ( 
+        10.0.0.0/8 or 
+        100.64.0.0/10 or 
+        127.0.0.0/8 or 
+        169.254.0.0/16 or 
+        172.16.0.0/12 or 
+        192.0.0.0/24 or 
+        192.0.0.0/29 or 
+        192.0.0.10/32 or 
+        192.0.0.170/32 or 
+        192.0.0.171/32 or 
+        192.0.0.8/32 or 
+        192.0.0.9/32 or 
+        192.0.2.0/24 or 
+        192.168.0.0/16 or 
+        192.175.48.0/24 or 
+        192.31.196.0/24 or 
+        192.52.193.0/24 or 
+        192.88.99.0/24 or 
+        198.18.0.0/15 or 
+        198.51.100.0/24 or 
+        203.0.113.0/24 or 
+        224.0.0.0/4 or 
+        240.0.0.0/4 or 
+        "::1" or 
+        "FE80::/10" or 
+        "FF00::/8")
 '''

+
 [[rule.threat]]
 framework = "MITRE ATT&CK"
-
 [[rule.threat.technique]]
 id = "T1071"
 name = "Application Layer Protocol"
 reference = "https://attack.mitre.org/techniques/T1071/"

+
 [rule.threat.tactic]
 id = "TA0011"
 name = "Command and Control"
@@ -92,7 +105,8 @@ reference = "https://attack.mitre.org/tactics/TA0011/"
 [rule.new_terms]
 field = "new_terms_fields"
 value = ["destination.ip", "process.executable"]
-
 [[rule.new_terms.history_window_start]]
 field = "history_window_start"
 value = "now-2d"
+
+
@@ -29,10 +29,13 @@ timestamp_override = "event.ingested"
 type = "query"

 query = '''
-event.category:file and host.os.type:linux and event.type:change and
- process.name:* and
- (file.path:(/usr/sbin/sshd or /usr/bin/ssh or /usr/bin/sftp or /usr/bin/scp) or file.name:libkeyutils.so) and
- not process.name:("dpkg" or "yum" or "dnf" or "dnf-automatic")
+event.category:file and host.os.type:linux and event.type:change and 
+  process.name:(* and not (dnf or dnf-automatic or dpkg or yum)) and 
+  (file.path:(/usr/bin/scp or 
+                /usr/bin/sftp or 
+                /usr/bin/ssh or 
+                /usr/sbin/sshd) or 
+  file.name:libkeyutils.so)
 '''


@@ -48,7 +51,6 @@ reference = "https://attack.mitre.org/techniques/T1543/"
 id = "TA0003"
 name = "Persistence"
 reference = "https://attack.mitre.org/tactics/TA0003/"
-
 [[rule.threat]]
 framework = "MITRE ATT&CK"
 [[rule.threat.technique]]
@@ -61,21 +63,8 @@ reference = "https://attack.mitre.org/techniques/T1556/"
 id = "TA0006"
 name = "Credential Access"
 reference = "https://attack.mitre.org/tactics/TA0006/"
-
-
-
-
 [[rule.threat]]
 framework = "MITRE ATT&CK"
-[[rule.threat.technique]]
-id = "T1563"
-name = "Remote Service Session Hijacking"
-reference = "https://attack.mitre.org/techniques/T1563/"
-[[rule.threat.technique.subtechnique]]
-id = "T1563.001"
-name = "SSH Hijacking"
-reference = "https://attack.mitre.org/techniques/T1563/001/"
-
 [[rule.threat.technique]]
 id = "T1021"
 name = "Remote Services"
@@ -85,7 +74,20 @@ id = "T1021.004"
 name = "SSH"
 reference = "https://attack.mitre.org/techniques/T1021/004/"

+
+[[rule.threat.technique]]
+id = "T1563"
+name = "Remote Service Session Hijacking"
+reference = "https://attack.mitre.org/techniques/T1563/"
+[[rule.threat.technique.subtechnique]]
+id = "T1563.001"
+name = "SSH Hijacking"
+reference = "https://attack.mitre.org/techniques/T1563/001/"
+
+
+
 [rule.threat.tactic]
 id = "TA0008"
 name = "Lateral Movement"
 reference = "https://attack.mitre.org/tactics/TA0008/"
+
@@ -4,16 +4,16 @@ integration = ["endpoint"]
 maturity = "production"
 min_stack_comments = "New fields added: required_fields, related_integrations, setup, New Term"
 min_stack_version = "8.6.0"
-updated_date = "2023/06/09"
+updated_date = "2023/06/22"

 [rule]
 author = ["Elastic"]
 description = """
-This rule monitors the creation of shared object files by previously unknown processes. The creation of a shared object 
-file involves compiling code into a dynamically linked library that can be loaded by other programs at runtime. While 
-this process is typically used for legitimate purposes, malicious actors can leverage shared object files to execute 
-unauthorized code, inject malicious functionality into legitimate processes, or bypass security controls. This allows 
-malware to persist on the system, evade detection, and potentially compromise the integrity and confidentiality of the 
+This rule monitors the creation of shared object files by previously unknown processes. The creation of a shared object
+file involves compiling code into a dynamically linked library that can be loaded by other programs at runtime. While
+this process is typically used for legitimate purposes, malicious actors can leverage shared object files to execute
+unauthorized code, inject malicious functionality into legitimate processes, or bypass security controls. This allows
+malware to persist on the system, evade detection, and potentially compromise the integrity and confidentiality of the
 affected system and its data.
 """
 from = "now-9m"
@@ -21,9 +21,7 @@ index = ["logs-endpoint.events.*", "endgame-*"]
 language = "kuery"
 license = "Elastic License v2"
 name = "Shared Object Created or Changed by Previously Unknown Process"
-references = [
-    "https://threatpost.com/sneaky-malware-backdoors-linux/180158/"
-]
+references = ["https://threatpost.com/sneaky-malware-backdoors-linux/180158/"]
 risk_score = 47
 rule_id = "aebaa51f-2a91-4f6a-850b-b601db2293f4"
 severity = "medium"
@@ -32,23 +30,25 @@ timestamp_override = "event.ingested"
 type = "new_terms"

 query = '''
-host.os.type : "linux" and event.action:("creation" or "file_create_event" or "rename" or "file_rename_event") and 
-file.path : (/usr/lib/* or /dev/shm/*) and file.extension : "so" and process.name : * and not 
-process.name : ("dpkg" or "dockerd" or "rpm" or "snapd" or "5")
+host.os.type:linux and event.action:(creation or file_create_event or file_rename_event or rename) and 
+file.path:(/dev/shm/* or /usr/lib/*) and file.extension:so and 
+process.name:(* and not (5 or dockerd or dpkg or rpm or snapd))
 '''

+
 [[rule.threat]]
 framework = "MITRE ATT&CK"
 [[rule.threat.technique]]
 id = "T1574"
 name = "Hijack Execution Flow"
 reference = "https://attack.mitre.org/techniques/T1574/"
-
 [[rule.threat.technique.subtechnique]]
 id = "T1574.006"
 name = "Dynamic Linker Hijacking"
 reference = "https://attack.mitre.org/techniques/T1574/006/"

+
+
 [rule.threat.tactic]
 id = "TA0003"
 name = "Persistence"
@@ -57,7 +57,8 @@ reference = "https://attack.mitre.org/tactics/TA0003/"
 [rule.new_terms]
 field = "new_terms_fields"
 value = ["file.path", "process.name"]
-
 [[rule.new_terms.history_window_start]]
 field = "history_window_start"
 value = "now-7d"
+
+