[New Rule] Initial converted EQL rules (#304)

* 18 converted eql rules (not all prod)

detection_rules/ecs.py

@@ -174,6 +174,7 @@ class KqlSchema2Eql(eql.Schema):
         "double": eql.types.TypeHint.Numeric,
         "long": eql.types.TypeHint.Numeric,
         "short": eql.types.TypeHint.Numeric,
+        "boolean": eql.types.TypeHint.Boolean,
     }
 
     def __init__(self, kql_schema):

detection_rules/rule.py

@@ -152,7 +152,7 @@ class Rule(object):
 
     def normalize(self, indent=2):
         """Normalize the (api only) contents and return a serialized dump of it."""
-        return json.dumps(nested_normalize(self.contents), sort_keys=True, indent=indent)
+        return json.dumps(nested_normalize(self.contents, eql_rule=self.type == 'eql'), sort_keys=True, indent=indent)
 
     def get_path(self):
         """Wrapper around getting path."""
@@ -185,7 +185,10 @@ class Rule(object):
 
         schema_cls.validate(contents, role=self.type)
 
-        if query and self.query is not None:
+        skip_query_validation = self.metadata['maturity'] == 'development' and \
+            self.metadata.get('query_schema_validation') is False
+
+        if query and self.query is not None and not skip_query_validation:
             ecs_versions = self.metadata.get('ecs_version')
             indexes = self.contents.get("index", [])
 

detection_rules/rule_formatter.py

@@ -24,21 +24,25 @@ def cleanup_whitespace(val):
     return val
 
 
-def nested_normalize(d, skip_cleanup=False):
+def nested_normalize(d, skip_cleanup=False, eql_rule=False):
     if isinstance(d, str):
         return d if skip_cleanup else cleanup_whitespace(d)
     elif isinstance(d, list):
-        return [nested_normalize(val) for val in d]
+        return [nested_normalize(val, eql_rule=eql_rule) for val in d]
     elif isinstance(d, dict):
         for k, v in d.items():
             if k == 'query':
                 # TODO: the linter still needs some work, but once up to par, uncomment to implement - kql.lint(v)
-                d.update({k: nested_normalize(v)})
+                if eql_rule:
+                    # do not normalize eql queries
+                    d.update({k: v})
+                else:
+                    d.update({k: nested_normalize(v)})
             elif k in CurrentSchema.markdown_fields():
                 # let these maintain newlines and whitespace for markdown support
-                d.update({k: nested_normalize(v, skip_cleanup=True)})
+                d.update({k: nested_normalize(v, skip_cleanup=True, eql_rule=eql_rule)})
             else:
-                d.update({k: nested_normalize(v)})
+                d.update({k: nested_normalize(v, eql_rule=eql_rule)})
         return d
     else:
         return d

detection_rules/schemas/base.py

@@ -82,6 +82,7 @@ class TomlMetadata(GenericSchema):
     related_endpoint_rules = jsl.ArrayField(jsl.ArrayField(jsl.StringField(), min_items=2, max_items=2),
                                             required=False)
     updated_date = jsl.StringField(required=True, pattern=DATE_PATTERN, default=time.strftime('%Y/%m/%d'))
+    query_schema_validation = jsl.BooleanField(required=False)
 
 
 class BaseApiSchema(GenericSchema):

rules/windows/c2_installutil_beacon.toml

@@ -0,0 +1,44 @@
+[metadata]
+creation_date = "2020/09/02"
+ecs_version = ["1.6.0"]
+maturity = "production"
+updated_date = "2020/09/02"
+
+[rule]
+author = ["Elastic"]
+description = """
+Identifies InstallUtil.exe making outbound network connections. This may indicate adversarial activity as InstallUtil is
+often leveraged by adversaries to execute code and evade detection.
+"""
+from = "now-9m"
+index = ["logs-endpoint.events.*", "winlogbeat-*"]
+language = "eql"
+license = "Elastic License"
+name = "InstallUtil Process Making Network Connections"
+risk_score = 21
+rule_id = "a13167f1-eec2-4015-9631-1fee60406dcf"
+severity = "medium"
+tags = ["Elastic", "Windows"]
+type = "eql"
+
+query = '''
+/* this can be done without a sequence however, this does include more info on the process */
+
+sequence by process.entity_id
+  [process where event.type in ("start", "process_started") and process.name == "installutil.exe"]
+  [network where event.type == "connection" and process.name == "installutil.exe" and network.direction == "outgoing"]
+'''
+
+
+[[rule.threat]]
+framework = "MITRE ATT&CK"
+[[rule.threat.technique]]
+id = "T1118"
+name = "InstallUtil"
+reference = "https://attack.mitre.org/techniques/T1118/"
+
+
+[rule.threat.tactic]
+id = "TA0005"
+name = "Defense Evasion"
+reference = "https://attack.mitre.org/tactics/TA0005/"

rules/windows/c2_msbuild_beacon_sequence.toml

@@ -0,0 +1,43 @@
+[metadata]
+creation_date = "2020/09/02"
+ecs_version = ["1.6.0"]
+maturity = "production"
+updated_date = "2020/09/02"
+
+[rule]
+author = ["Elastic"]
+description = """
+Identifies MsBuild.exe making outbound network connections. This may indicate adversarial activity as MsBuild is often
+leveraged by adversaries to execute code and evade detection.
+"""
+from = "now-9m"
+index = ["logs-endpoint.events.*", "winlogbeat-*"]
+language = "eql"
+license = "Elastic License"
+name = "MsBuild Network Connection Sequence"
+risk_score = 21
+rule_id = "9dc6ed5d-62a9-4feb-a903-fafa1d33b8e9"
+severity = "medium"
+tags = ["Elastic", "Windows"]
+type = "eql"
+
+query = '''
+sequence by process.entity_id
+  [process where event.type in ("start", "process_started") and process.name == "MSBuild.exe"]
+  [network where process.name == "MSBuild.exe" and
+     not (destination.address == "127.0.0.1" and source.address == "127.0.0.1")]
+'''
+
+
+[[rule.threat]]
+framework = "MITRE ATT&CK"
+[[rule.threat.technique]]
+id = "T1127"
+name = "Trusted Developer Utilities Proxy Execution"
+reference = "https://attack.mitre.org/techniques/T1127/"
+
+
+[rule.threat.tactic]
+id = "TA0005"
+name = "Defense Evasion"
+reference = "https://attack.mitre.org/tactics/TA0005/"

rules/windows/c2_mshta_beacon.toml

@@ -0,0 +1,46 @@
+[metadata]
+creation_date = "2020/09/02"
+ecs_version = ["1.6.0"]
+maturity = "production"
+updated_date = "2020/09/02"
+
+[rule]
+author = ["Elastic"]
+description = """
+Identifies Mshta.exe making outbound network connections. This may indicate adversarial activity as Mshta is often
+leveraged by adversaries to execute malicious scripts and evade detection.
+"""
+from = "now-9m"
+index = ["logs-endpoint.events.*", "winlogbeat-*"]
+language = "eql"
+license = "Elastic License"
+name = "Mshta Making Network Connections"
+risk_score = 21
+rule_id = "c2d90150-0133-451c-a783-533e736c12d7"
+severity = "medium"
+tags = ["Elastic", "Windows"]
+type = "eql"
+
+query = '''
+sequence by process.entity_id with maxspan=2h
+  [process where event.type in ("start", "process_started") and process.name == "mshta.exe" and
+     process.parent.name != "Microsoft.ConfigurationManagement.exe" and
+     process.parent.executable not in ("C:\\Amazon\\Amazon Assistant\\amazonAssistantService.exe",
+                                       "C:\\TeamViewer\\TeamViewer.exe") and
+     process.args != "ADSelfService_Enroll.hta"]
+  [network where process.name == "mshta.exe"]
+'''
+
+
+[[rule.threat]]
+framework = "MITRE ATT&CK"
+[[rule.threat.technique]]
+id = "T1170"
+name = "Mshta"
+reference = "https://attack.mitre.org/techniques/T1170/"
+
+
+[rule.threat.tactic]
+id = "TA0005"
+name = "Defense Evasion"
+reference = "https://attack.mitre.org/tactics/TA0005/"

rules/windows/c2_msxsl_beacon.toml

@@ -0,0 +1,42 @@
+[metadata]
+creation_date = "2020/09/02"
+ecs_version = ["1.6.0"]
+maturity = "production"
+updated_date = "2020/09/02"
+
+[rule]
+author = ["Elastic"]
+description = """
+Identifies MsXsl.exe making outbound network connections. This may indicate adversarial activity as MsXsl is often
+leveraged by adversaries to execute malicious scripts and evade detection.
+"""
+from = "now-9m"
+index = ["logs-endpoint.events.*", "winlogbeat-*"]
+language = "eql"
+license = "Elastic License"
+name = "MsXsl Making Network Connections"
+risk_score = 21
+rule_id = "870d1753-1078-403e-92d4-735f142edcca"
+severity = "medium"
+tags = ["Elastic", "Windows"]
+type = "eql"
+
+query = '''
+sequence by process.entity_id
+  [process where event.type in ("start", "process_started") and process.name == "msxsl.exe"]
+  [network where event.type == "connection" and process.name == "msxsl.exe" and network.direction == "outgoing"]
+'''
+
+
+[[rule.threat]]
+framework = "MITRE ATT&CK"
+[[rule.threat.technique]]
+id = "T1220"
+name = "XSL Script Processing"
+reference = "https://attack.mitre.org/techniques/T1220/"
+
+
+[rule.threat.tactic]
+id = "TA0005"
+name = "Defense Evasion"
+reference = "https://attack.mitre.org/tactics/TA0005/"

rules/windows/c2_network_connection_from_windows_binary.toml

@@ -0,0 +1,84 @@
+[metadata]
+creation_date = "2020/09/02"
+ecs_version = ["1.6.0"]
+maturity = "production"
+updated_date = "2020/09/02"
+
+[rule]
+author = ["Elastic"]
+description = """
+Identifies network activity from unexpected system applications. This may indicate adversarial activity as these
+applications are often leveraged by adversaries to execute code and evade detection.
+"""
+from = "now-9m"
+index = ["logs-endpoint.events.*", "winlogbeat-*"]
+language = "eql"
+license = "Elastic License"
+name = "Unusual Network Activity from a Windows System Binary"
+risk_score = 21
+rule_id = "1fe3b299-fbb5-4657-a937-1d746f2c711a"
+severity = "medium"
+tags = ["Elastic", "Windows"]
+type = "eql"
+
+query = '''
+sequence by process.entity_id with maxspan=5m
+  [process where event.type in ("start", "process_started") and
+
+     /* known applocker bypasses */
+     process.name in ("bginfo.exe",
+                      "cdb.exe",
+                      "control.exe",
+                      "cmstp.exe",
+                      "csi.exe",
+                      "dnx.exe",
+                      "fsi.exe",
+                      "ieexec.exe",
+                      "iexpress.exe",
+                      "installutil.exe",
+                      "Microsoft.Workflow.Compiler.exe",
+                      "MSBuild.exe",
+                      "msdt.exe",
+                      "mshta.exe",
+                      "msiexec.exe",
+                      "msxsl.exe",
+                      "odbcconf.exe",
+                      "rcsi.exe",
+                      "regsvr32.exe",
+                      "xwizard.exe")]
+  [network where event.type == "connection" and
+     process.name in ("bginfo.exe",
+                      "cdb.exe",
+                      "control.exe",
+                      "cmstp.exe",
+                      "csi.exe",
+                      "dnx.exe",
+                      "fsi.exe",
+                      "ieexec.exe",
+                      "iexpress.exe",
+                      "installutil.exe",
+                      "Microsoft.Workflow.Compiler.exe",
+                      "MSBuild.exe",
+                      "msdt.exe",
+                      "mshta.exe",
+                      "msiexec.exe",
+                      "msxsl.exe",
+                      "odbcconf.exe",
+                      "rcsi.exe",
+                      "regsvr32.exe",
+                      "xwizard.exe")]
+'''
+
+
+[[rule.threat]]
+framework = "MITRE ATT&CK"
+[[rule.threat.technique]]
+id = "T1127"
+name = "Trusted Developer Utilities Proxy Execution"
+reference = "https://attack.mitre.org/techniques/T1127/"
+
+
+[rule.threat.tactic]
+id = "TA0005"
+name = "Defense Evasion"
+reference = "https://attack.mitre.org/tactics/TA0005/"

rules/windows/c2_reg_beacon.toml

@@ -0,0 +1,45 @@
+[metadata]
+creation_date = "2020/09/02"
+ecs_version = ["1.6.0"]
+maturity = "production"
+updated_date = "2020/09/02"
+
+[rule]
+author = ["Elastic"]
+description = """
+Identifies registration utilities making outbound network connections. This includes regsvcs, regasm, and regsvr32. This
+may indicate adversarial activity as these tools are often leveraged by adversaries to execute code and evade detection.
+"""
+from = "now-9m"
+index = ["logs-endpoint.events.*", "winlogbeat-*"]
+language = "eql"
+license = "Elastic License"
+name = "Registration Tool Making Network Connections"
+risk_score = 21
+rule_id = "6d3456a5-4a42-49d1-aaf2-7b1fd475b2c6"
+severity = "medium"
+tags = ["Elastic", "Windows"]
+type = "eql"
+
+query = '''
+sequence by process.entity_id
+  [process where event.type in ("start", "process_started") and
+     process.name in ("regasm.exe", "regsvcs.exe", "regsvr32.exe")]
+  [network where event.type == "connection" and process.name in ("regasm.exe", "regsvcs.exe", "regsvr32.exe")]
+until
+  [process where event.type == "end" and process.name in ("regasm.exe", "regsvcs.exe", "regsvr32.exe")]
+'''
+
+
+[[rule.threat]]
+framework = "MITRE ATT&CK"
+[[rule.threat.technique]]
+id = "T1121"
+name = "Regsvcs/Regasm"
+reference = "https://attack.mitre.org/techniques/T1121/"
+
+
+[rule.threat.tactic]
+id = "TA0005"
+name = "Defense Evasion"
+reference = "https://attack.mitre.org/tactics/TA0005/"

rules/windows/c2_rundll32_sequence.toml

@@ -0,0 +1,49 @@
+[metadata]
+creation_date = "2020/09/02"
+ecs_version = ["1.6.0"]
+maturity = "production"
+updated_date = "2020/09/02"
+
+[rule]
+author = ["Elastic"]
+description = """
+Identifies unusual instances of Rundll32.exe making outbound network connections. This may indicate adversarial activity
+and may identify malicious DLLs.
+"""
+from = "now-9m"
+index = ["logs-endpoint.events.*", "winlogbeat-*"]
+language = "eql"
+license = "Elastic License"
+name = "Unusual Network Connection Sequence via RunDLL32"
+risk_score = 21
+rule_id = "2b347f66-6739-4ae3-bd94-195036dde8b3"
+severity = "low"
+tags = ["Elastic", "Windows"]
+type = "eql"
+
+query = '''
+sequence by process.entity_id with maxspan=2h
+  [process where event.type in ("start", "process_started") and
+     (process.name == "rundll32.exe" or process.pe.original_file_name == "rundll32.exe") and
+
+     /* zero arguments excluding the binary itself (and accounting for when the binary may not be logged in args) */
+     ((process.args == "rundll32.exe" and process.args_count == 1) or
+      (process.args != "rundll32.exe" and process.args_count == 0))]
+
+  [network where event.type == "connection" and
+     (process.name == "rundll32.exe" or process.pe.original_file_name == "rundll32.exe")]
+'''
+
+
+[[rule.threat]]
+framework = "MITRE ATT&CK"
+[[rule.threat.technique]]
+id = "T1085"
+name = "Rundll32"
+reference = "https://attack.mitre.org/techniques/T1085/"
+
+
+[rule.threat.tactic]
+id = "TA0005"
+name = "Defense Evasion"
+reference = "https://attack.mitre.org/tactics/TA0005/"

rules/windows/escalation_uac_sdclt.toml

@@ -0,0 +1,51 @@
+[metadata]
+creation_date = "2020/09/02"
+ecs_version = ["1.6.0"]
+maturity = "production"
+updated_date = "2020/09/02"
+
+[rule]
+author = ["Elastic"]
+description = """
+Identifies User Account Control (UAC) bypass via sdclt.exe. Attackers bypass UAC to stealthily execute code with
+elevated permissions.
+"""
+from = "now-9m"
+index = ["logs-endpoint.events.*", "winlogbeat-*"]
+language = "eql"
+license = "Elastic License"
+name = "Bypass UAC via Sdclt"
+risk_score = 21
+rule_id = "9b54e002-034a-47ac-9307-ad12c03fa900"
+severity = "high"
+tags = ["Elastic", "Windows"]
+type = "eql"
+
+query = '''
+sequence with maxspan=1m
+  [process where event.type in ("start", "process_started") and process.name == "sdclt.exe" and
+     /* process.code_signature.* fields need to be populated for 7.10 */
+     process.code_signature.subject_name == "Microsoft Corporation" and process.code_signature.trusted == true and
+     process.args == "/kickoffelev"
+  ] by process.entity_id
+  [process where event.type in ("start", "process_started") and process.parent.name == "sdclt.exe" and
+     process.executable not in ("C:\\Windows\\System32\\sdclt.exe",
+                                "C:\\Windows\\System32\\control.exe",
+                                "C:\\Windows\\SysWOW64\\sdclt.exe",
+                                "C:\\Windows\\SysWOW64\\control.exe")
+  ] by process.parent.entity_id
+'''
+
+
+[[rule.threat]]
+framework = "MITRE ATT&CK"
+[[rule.threat.technique]]
+id = "T1088"
+name = "Bypass User Account Control"
+reference = "https://attack.mitre.org/techniques/T1088/"
+
+
+[rule.threat.tactic]
+id = "TA0004"
+name = "Privilege Escalation"
+reference = "https://attack.mitre.org/tactics/TA0004/"

rules/windows/evasion_rundll32_no_arguments.toml

@@ -0,0 +1,51 @@
+[metadata]
+creation_date = "2020/09/02"
+ecs_version = ["1.6.0"]
+maturity = "production"
+updated_date = "2020/09/02"
+
+[rule]
+author = ["Elastic"]
+description = """
+Identifies child processes of unusual instances of RunDLL32 where the command line parameters were suspicious. Misuse of
+RunDLL32 could indicate malicious activity.
+"""
+from = "now-9m"
+index = ["logs-endpoint.events.*", "winlogbeat-*"]
+language = "eql"
+license = "Elastic License"
+name = "Unusual Child Processes of RunDLL32"
+risk_score = 21
+rule_id = "f036953a-4615-4707-a1ca-dc53bf69dcd5"
+severity = "high"
+tags = ["Elastic", "Windows"]
+type = "eql"
+
+query = '''
+sequence with maxspan=1h
+  [process where event.type in ("start", "process_started") and
+     (process.name == "rundll32.exe" or process.pe.original_file_name == "rundll32.exe") and
+
+     /* zero arguments excluding the binary itself (and accounting for when the binary may not be logged in args) */
+     ((process.args == "rundll32.exe" and process.args_count == 1) or
+      (process.args != "rundll32.exe" and process.args_count == 0))
+
+  ] by process.entity_id
+  [process where event.type in ("start", "process_started") and
+     (process.name == "rundll32.exe" or process.pe.original_file_name == "rundll32.exe")
+  ] by process.parent.entity_id
+'''
+
+
+[[rule.threat]]
+framework = "MITRE ATT&CK"
+[[rule.threat.technique]]
+id = "T1085"
+name = "Rundll32"
+reference = "https://attack.mitre.org/techniques/T1085/"
+
+
+[rule.threat.tactic]
+id = "TA0005"
+name = "Defense Evasion"
+reference = "https://attack.mitre.org/tactics/TA0005/"

rules/windows/evasion_suspicious_scrobj_load.toml

@@ -0,0 +1,52 @@
+[metadata]
+creation_date = "2020/09/02"
+ecs_version = ["1.6.0"]
+maturity = "production"
+updated_date = "2020/09/02"
+
+[rule]
+author = ["Elastic"]
+description = """
+Identifies scrobj.dll loaded into unusual Microsoft processes. This usually means a malicious scriptlet is being
+executed in the target process.
+"""
+from = "now-9m"
+index = ["logs-endpoint.events.*", "winlogbeat-*"]
+language = "eql"
+license = "Elastic License"
+name = "Windows Suspicious Script Object Execution"
+risk_score = 21
+rule_id = "4ed678a9-3a4f-41fb-9fea-f85a6e0a0dff"
+severity = "medium"
+tags = ["Elastic", "Windows"]
+type = "eql"
+
+query = '''
+sequence by process.entity_id with maxspan=2m
+  [process where event.type in ("start", "process_started") and
+     /* process.code_signature.* fields need to be populated for 7.10 */
+     process.code_signature.subject_name == "Microsoft Corporation" and process.code_signature.trusted == true and
+     process.name not in ("cscript.exe",
+                          "iexplore.exe",
+                          "MicrosoftEdge.exe",
+                          "msiexec.exe",
+                          "smartscreen.exe",
+                          "taskhostw.exe",
+                          "w3wp.exe",
+                          "wscript.exe")]
+  [library where event.type == "start" and file.name == "scrobj.dll"]
+'''
+
+
+[[rule.threat]]
+framework = "MITRE ATT&CK"
+[[rule.threat.technique]]
+id = "T1064"
+name = "Scripting"
+reference = "https://attack.mitre.org/techniques/T1064/"
+
+
+[rule.threat.tactic]
+id = "TA0005"
+name = "Defense Evasion"
+reference = "https://attack.mitre.org/tactics/TA0005/"

rules/windows/evasion_suspicious_wmi_script.toml

@@ -0,0 +1,48 @@
+[metadata]
+creation_date = "2020/09/02"
+ecs_version = ["1.6.0"]
+maturity = "production"
+updated_date = "2020/09/02"
+
+[rule]
+author = ["Elastic"]
+description = """
+Identifies WMIC whitelisting bypass techniques by alerting on suspicious execution of scripts. When WMIC loads scripting
+libraries it may be indicative of a whitelist bypass.
+"""
+from = "now-9m"
+index = ["logs-endpoint.events.*", "winlogbeat-*"]
+language = "eql"
+license = "Elastic License"
+name = "Suspicious WMIC XSL Script Execution"
+risk_score = 21
+rule_id = "7f370d54-c0eb-4270-ac5a-9a6020585dc6"
+severity = "medium"
+tags = ["Elastic", "Windows"]
+type = "eql"
+
+query = '''
+/* lots of wildcards in the args
+   need to verify args cleanup is accurate
+*/
+
+sequence by process.entity_id with maxspan=2m
+[process where event.type in ("start", "process_started") and
+   (process.name == "wmic.exe" or process.pe.original_file_name == "wmic.exe") and
+   wildcard(process.args, "format*:*", "/format*:*", "*-format*:*") and
+   not process.args in ("/format:table", "/format:table") or wildcard(process.args, "format*:*")]
+[library where event.type == "start" and file.name in ("jscript.dll", "vbscript.dll")]
+'''
+
+
+[[rule.threat]]
+framework = "MITRE ATT&CK"
+[[rule.threat.technique]]
+id = "T1220"
+name = "XSL Script Processing"
+reference = "https://attack.mitre.org/techniques/T1220/"
+
+[rule.threat.tactic]
+id = "TA0005"
+name = "Defense Evasion"
+reference = "https://attack.mitre.org/tactics/TA0005/"

rules/windows/execution_downloaded_shortcut_files.toml

@@ -0,0 +1,59 @@
+[metadata]
+creation_date = "2020/09/02"
+ecs_version = ["1.6.0"]
+maturity = "development"
+query_schema_validation = false
+updated_date = "2020/09/02"
+
+[rule]
+author = ["Elastic"]
+description = """
+Identifies .lnk shortcut file downloaded from outside the local network. These shortcut files are commonly used in
+phishing campaigns.
+"""
+from = "now-9m"
+index = ["logs-endpoint.events.*", "winlogbeat-*"]
+language = "eql"
+license = "Elastic License"
+name = "Downloaded Shortcut Files"
+risk_score = 21
+rule_id = "6b1fd8e8-cefe-444c-bc4d-feaa2c497347"
+severity = "low"
+tags = ["Elastic", "Windows"]
+type = "eql"
+
+query = '''
+/* leaving in development pending `file.Ext.windows.zone_identifier` landing in ECS then endpoint */
+
+sequence by process.entity_id with maxspan=2s
+                                           /* file.extension added to endpoint fields for 7.10 */
+  [file where event.type == "creation" and file.extension == "lnk"]
+                                           /* not sure yet how the update will capture ADS */
+  [file where event.type == "creation" and file.extension == "lnk:Zone.Identifier" and
+     /* non-ECS field - may disqualify conversion */
+     file.Ext.windows.zone_identifier > 1]
+'''
+
+
+[[rule.threat]]
+framework = "MITRE ATT&CK"
+[[rule.threat.technique]]
+id = "T1193"
+name = "Spearphishing Attachment"
+reference = "https://attack.mitre.org/techniques/T1193/"
+
+[[rule.threat.technique]]
+id = "T1192"
+name = "Spearphishing Link"
+reference = "https://attack.mitre.org/techniques/T1192/"
+
+[[rule.threat.technique]]
+id = "T1204"
+name = "User Execution"
+reference = "https://attack.mitre.org/techniques/T1204/"
+
+
+[rule.threat.tactic]
+id = "TA0002"
+name = "Execution"
+reference = "https://attack.mitre.org/tactics/TA0002/"

rules/windows/execution_downloaded_url_file.toml

@@ -0,0 +1,63 @@
+[metadata]
+creation_date = "2020/09/02"
+ecs_version = ["1.6.0"]
+maturity = "development"
+query_schema_validation = false
+updated_date = "2020/09/02"
+
+[rule]
+author = ["Elastic"]
+description = """
+Identifies .url shortcut files downloaded from outside the local network. These shortcut files are commonly used in
+phishing campaigns.
+"""
+from = "now-9m"
+index = ["logs-endpoint.events.*", "winlogbeat-*"]
+language = "eql"
+license = "Elastic License"
+name = "Downloaded URL Files"
+risk_score = 21
+rule_id = "cd82e3d6-1346-4afd-8f22-38388bbf34cb"
+severity = "medium"
+tags = ["Elastic", "Windows"]
+type = "eql"
+
+query = '''
+/* leaving in development pending `file.Ext.windows.zone_identifier` landing in ECS then endpoint */
+
+sequence by process.entity_id with maxspan=2s
+  [file where event.type == "creation" and file.extension == "url" and
+     not process.name == "explorer.exe"]
+  [file where event.type == "creation" and file.extension == "url:Zone.Identifier" and
+      /* non-ECS field - may disqualify conversion */
+     file.Ext.windows.zone_identifier > 1 and not process.name == "explorer.exe"]
+'''
+
+
+[[rule.threat]]
+framework = "MITRE ATT&CK"
+[[rule.threat.technique]]
+id = "T1193"
+name = "Spearphishing Attachment"
+reference = "https://attack.mitre.org/techniques/T1193/"
+
+[[rule.threat.technique]]
+id = "T1192"
+name = "Spearphishing Link"
+reference = "https://attack.mitre.org/techniques/T1192/"
+
+[[rule.threat.technique]]
+id = "T1204"
+name = "User Execution"
+reference = "https://attack.mitre.org/techniques/T1204/"
+
+[[rule.threat.technique]]
+id = "T1064"
+name = "Scripting"
+reference = "https://attack.mitre.org/techniques/T1064/"
+
+
+[rule.threat.tactic]
+id = "TA0002"
+name = "Execution"
+reference = "https://attack.mitre.org/tactics/TA0002/"

rules/windows/execution_ms_office_written_file.toml

@@ -0,0 +1,61 @@
+[metadata]
+creation_date = "2020/09/02"
+ecs_version = ["1.6.0"]
+maturity = "production"
+updated_date = "2020/09/02"
+
+[rule]
+author = ["Elastic"]
+description = """
+Identifies an executable created by a Microsoft Office application and subsequently executed. These processes are often
+launched via scripts inside documents or during exploitation of MS Office applications.
+"""
+from = "now-9m"
+index = ["logs-endpoint.events.*", "winlogbeat-*"]
+language = "eql"
+license = "Elastic License"
+name = "Execution of File Written or Modified by Microsoft Office"
+risk_score = 21
+rule_id = "0d8ad79f-9025-45d8-80c1-4f0cd3c5e8e5"
+severity = "high"
+tags = ["Elastic", "Windows"]
+type = "eql"
+
+query = '''
+sequence with maxspan=2h
+  [file where event.type != "delete" and file.extension == "exe" and
+     process.name in ("winword.exe",
+                      "excel.exe",
+                      "outlook.exe",
+                      "powerpnt.exe",
+                      "eqnedt32.exe",
+                      "fltldr.exe",
+                      "mspub.exe",
+                      "msaccess.exe")
+  ] by host.id, file.path
+  [process where event.type in ("start", "process_started")] by host.id, process.executable
+'''
+
+
+[[rule.threat]]
+framework = "MITRE ATT&CK"
+[[rule.threat.technique]]
+id = "T1064"
+name = "Scripting"
+reference = "https://attack.mitre.org/techniques/T1064/"
+
+[[rule.threat.technique]]
+id = "T1192"
+name = "Spearphishing Link"
+reference = "https://attack.mitre.org/techniques/T1192/"
+
+[[rule.threat.technique]]
+id = "T1193"
+name = "Spearphishing Attachment"
+reference = "https://attack.mitre.org/techniques/T1193/"
+
+
+[rule.threat.tactic]
+id = "TA0002"
+name = "Execution"
+reference = "https://attack.mitre.org/tactics/TA0002/"

rules/windows/execution_pdf_written_file.toml

@@ -0,0 +1,60 @@
+[metadata]
+creation_date = "2020/09/02"
+ecs_version = ["1.6.0"]
+maturity = "production"
+updated_date = "2020/09/02"
+
+[rule]
+author = ["Elastic"]
+description = """
+Identifies a suspicious file that was written by a PDF reader application and subsequently executed. These processes are
+often launched via exploitation of PDF applications.
+"""
+from = "now-9m"
+index = ["logs-endpoint.events.*", "winlogbeat-*"]
+language = "eql"
+license = "Elastic License"
+name = "Execution of File Written or Modified by PDF Reader"
+risk_score = 21
+rule_id = "1defdd62-cd8d-426e-a246-81a37751bb2b"
+severity = "high"
+tags = ["Elastic", "Windows"]
+type = "eql"
+
+query = '''
+sequence with maxspan=2h
+  [file where event.type != "delete" and file.extension == "exe" and
+     process.name in ("acrord32.exe", "rdrcef.exe", "foxitphantomPDF.exe", "foxitreader.exe") and
+     file.name not in ("foxitphantomPDF.exe",
+                       "FoxitPhantomPDFUpdater.exe",
+                       "foxitreader.exe",
+                       "FoxitReaderUpdater.exe",
+                       "acrord32.exe",
+                       "rdrcef.exe")
+  ] by host.id, file.path
+  [process where event.type in ("start", "process_started")] by host.id, process.executable
+'''
+
+
+[[rule.threat]]
+framework = "MITRE ATT&CK"
+[[rule.threat.technique]]
+id = "T1064"
+name = "Scripting"
+reference = "https://attack.mitre.org/techniques/T1064/"
+
+[[rule.threat.technique]]
+id = "T1192"
+name = "Spearphishing Link"
+reference = "https://attack.mitre.org/techniques/T1192/"
+
+[[rule.threat.technique]]
+id = "T1193"
+name = "Spearphishing Attachment"
+reference = "https://attack.mitre.org/techniques/T1193/"
+
+
+[rule.threat.tactic]
+id = "TA0002"
+name = "Execution"
+reference = "https://attack.mitre.org/tactics/TA0002/"

rules/windows/execution_wpad_exploitation.toml

@@ -0,0 +1,53 @@
+[metadata]
+creation_date = "2020/09/02"
+ecs_version = ["1.6.0"]
+maturity = "production"
+updated_date = "2020/09/02"
+
+[rule]
+author = ["Elastic"]
+description = """
+Identifies probable exploitation of the Web Proxy Auto-Discovery Protocol (WPAD) service. Attackers who have access to
+the local network or upstream DNS traffic can inject malicious JavaScript to the WPAD service which can lead to a full
+system compromise.
+"""
+from = "now-9m"
+index = ["logs-endpoint.events.*", "winlogbeat-*"]
+language = "eql"
+license = "Elastic License"
+name = "WPAD Service Exploit"
+risk_score = 21
+rule_id = "ec328da1-d5df-482b-866c-4a435692b1f3"
+severity = "high"
+tags = ["Elastic", "Windows"]
+type = "eql"
+
+query = '''
+/* preference would be to use user.sid rather than domain+name, once it is available in ECS + datasources */
+
+sequence with maxspan=5s
+  [process where event.type in ("start", "process_started") and process.name == "svchost.exe" and
+     user.domain == "NT AUTHORITY" and user.name == "LOCAL SERVICE"] by process.entity_id
+  [network where network.protocol == "dns" and process.name == "svchost.exe" and
+     dns.question.name == "wpad" and process.name == "svchost.exe"] by process.entity_id
+  [network where event.type == "connection" and process.name == "svchost.exe"
+     and network.direction == "outgoing" and destination.port == 80] by process.entity_id
+  [library where event.type == "start" and process.name == "svchost.exe" and
+     file.name == "jscript.dll" and process.name == "svchost.exe"] by process.entity_id
+  [process where event.type in ("start", "process_started") and
+     process.parent.name == "svchost.exe"] by process.parent.entity_id
+'''
+
+
+[[rule.threat]]
+framework = "MITRE ATT&CK"
+[[rule.threat.technique]]
+id = "T1068"
+name = "Exploitation for Privilege Escalation"
+reference = "https://attack.mitre.org/techniques/T1068/"
+
+
+[rule.threat.tactic]
+id = "TA0002"
+name = "Execution"
+reference = "https://attack.mitre.org/tactics/TA0002/"

rules/windows/lateral_movement_cmd_service.toml

@@ -0,0 +1,57 @@
+[metadata]
+creation_date = "2020/09/02"
+ecs_version = ["1.6.0"]
+maturity = "production"
+updated_date = "2020/09/02"
+
+[rule]
+author = ["Elastic"]
+description = """
+Identifies use of sc.exe to create, modify, or start services on remote hosts. This could be indicative of adversary
+lateral movement but will be noisy if commonly done by admins.
+"""
+from = "now-9m"
+index = ["logs-endpoint.events.*", "winlogbeat-*"]
+language = "eql"
+license = "Elastic License"
+name = "Service Command Lateral Movement"
+risk_score = 21
+rule_id = "d61cbcf8-1bc1-4cff-85ba-e7b21c5beedc"
+severity = "low"
+tags = ["Elastic", "Windows"]
+type = "eql"
+
+query = '''
+/* dependent on a wildcard for remote path */
+
+sequence by process.entity_id with maxspan=1m
+  [process where event.type in ("start", "process_started") and
+     (process.name == "sc.exe" or process.pe.original_file_name == "sc.exe") and
+     wildcard(process.args, "\\\\*") and wildcard(process.args, "binPath*", "binpath*") and
+     process.args in ("create", "config", "failure", "start")]
+  [network where event.type == "connection" and process.name == "sc.exe" and destination.address != "127.0.0.1"]
+'''
+
+
+[[rule.threat]]
+framework = "MITRE ATT&CK"
+[[rule.threat.technique]]
+id = "T1021"
+name = "Remote Services"
+reference = "https://attack.mitre.org/techniques/T1021/"
+
+[[rule.threat.technique]]
+id = "T1050"
+name = "New Service"
+reference = "https://attack.mitre.org/techniques/T1050/"
+
+[[rule.threat.technique]]
+id = "T1035"
+name = "Service Execution"
+reference = "https://attack.mitre.org/techniques/T1035/"
+
+
+[rule.threat.tactic]
+id = "TA0008"
+name = "Lateral Movement"
+reference = "https://attack.mitre.org/tactics/TA0008/"

rules/windows/persistence_app_compat_shim.toml

@@ -0,0 +1,46 @@
+[metadata]
+creation_date = "2020/09/02"
+ecs_version = ["1.6.0"]
+maturity = "production"
+updated_date = "2020/09/02"
+
+[rule]
+author = ["Elastic"]
+description = """
+Identifies the installation of custom Application Compatibility Shim databases. This Windows functionality has been
+abused by attackers to stealthily gain persistence and arbitrary code execution in legitimate Windows processes.
+"""
+from = "now-9m"
+index = ["logs-endpoint.events.*", "winlogbeat-*"]
+language = "eql"
+license = "Elastic License"
+name = "Installation of Custom Shim Databases"
+risk_score = 21
+rule_id = "c5ce48a6-7f57-4ee8-9313-3d0024caee10"
+severity = "medium"
+tags = ["Elastic", "Windows"]
+type = "eql"
+
+query = '''
+/* dependent on wildcard for registry.value */
+
+sequence by process.entity_id with maxspan=5m
+  [process where event.type in ("start", "process_started") and
+     not (process.name == "sdbinst.exe" and process.parent.name == "msiexec.exe")]
+  [registry where event.type in ("creation", "change") and
+     wildcard(registry.path, "HKLM\\SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\AppCompatFlags\\Custom\\*.sdb")]
+'''
+
+
+[[rule.threat]]
+framework = "MITRE ATT&CK"
+[[rule.threat.technique]]
+id = "T1138"
+name = "Application Shimming"
+reference = "https://attack.mitre.org/techniques/T1138/"
+
+
+[rule.threat.tactic]
+id = "TA0003"
+name = "Persistence"
+reference = "https://attack.mitre.org/tactics/TA0003/"

tests/test_toml_formatter.py

@@ -20,7 +20,7 @@ class TestRuleTomlFormatter(unittest.TestCase):
     with open(get_etc_path('test_toml.json'), 'r') as f:
         test_data = json.load(f)
 
-    def compare_formatted(self, data, callback=None):
+    def compare_formatted(self, data, callback=None, kwargs=None):
         """Compare formatted vs expected."""
         try:
             toml_write(copy.deepcopy(data), tmp_file)
@@ -36,7 +36,8 @@ class TestRuleTomlFormatter(unittest.TestCase):
             original = json.dumps(copy.deepcopy(data), sort_keys=True)
 
             if callback:
-                formatted_contents = callback(formatted_contents)
+                kwargs = kwargs or {}
+                formatted_contents = callback(formatted_contents, **kwargs)
 
             # callbacks such as nested normalize leave in line breaks, so this must be manually done
             query = formatted_contents.get('rule', {}).get('query')
@@ -71,4 +72,6 @@ class TestRuleTomlFormatter(unittest.TestCase):
         rules = rule_loader.load_rules().values()
 
         for rule in rules:
-            self.compare_formatted(rule.rule_format(formatted_query=False), callback=nested_normalize)
+            is_eql_rule = rule.type == 'eql'
+            self.compare_formatted(
+                rule.rule_format(formatted_query=False), callback=nested_normalize, kwargs={'eql_rule': is_eql_rule})