[New Rule] Elastic Endpoint and External Alerts (#42)

* Adds the Elastic Endpoint and External Alerts rules and required schema updates
* Optimizing queries to fix tests
* Apply PEP257 changes
* Apply suggestions from code review
* Update rules/cross-platform/external_alerts.toml
* Last fixes from review
* Fixing test for unrequired default
* Adding increased default max_signals to not interfere with testing
* Make promotions folder
* Refining Elastic Endpoint rule index

Co-authored-by: Ross Wolf <31489089+rw-access@users.noreply.github.com>
Co-authored-by: Justin Ibarra <brokensound77@users.noreply.github.com>

detection_rules/schema.py

@@ -16,6 +16,7 @@ DATE_PATTERN = r'\d{4}/\d{2}/\d{2}'
 VERSION_PATTERN = r'\d+\.\d+\.\d+'
 RULE_LEVELS = ['recommended', 'aggressive']
 MATURITY_LEVELS = ['development', 'testing', 'staged', 'production', 'deprecated']
+OPERATORS = ['equals']
 OS_OPTIONS = ['windows', 'linux', 'macos', 'solaris']  # need to verify with ecs
 INTERVAL_PATTERN = r'\d+[mshd]'
 MITRE_URL_PATTERN = r'https://attack.mitre.org/{type}/T[A-Z0-9]+/'
@@ -81,6 +82,23 @@ class Filters(jsl.Document):
     query = jsl.DocumentField(FilterQuery)
 
 
+class RiskScoreMapping(jsl.Document):
+    """Risk score mapping."""
+
+    field = jsl.StringField(required=True)
+    operator = jsl.StringField(required=False, enum=OPERATORS)
+    value = jsl.StringField(required=False)
+
+
+class SeverityMapping(jsl.Document):
+    """Severity mapping."""
+
+    field = jsl.StringField(required=True)
+    operator = jsl.StringField(required=False, enum=OPERATORS)
+    value = jsl.StringField(required=False)
+    severity = jsl.StringField(required=False)
+
+
 class ThreatTactic(jsl.Document):
     """Threat tactics."""
 
@@ -110,6 +128,7 @@ class SiemRuleApiSchema(jsl.Document):
 
     actions = jsl.ArrayField(required=False)
     author = jsl.ArrayField(jsl.StringField(default="Elastic"), required=True, min_items=1)
+    building_block_type = jsl.StringField(required=False)
     description = jsl.StringField(required=True)
     # api defaults to false if blank
     enabled = jsl.BooleanField(default=False, required=False)
@@ -127,13 +146,17 @@ class SiemRuleApiSchema(jsl.Document):
     # output_index = jsl.StringField(required=False)  # this is NOT allowed!
     references = jsl.ArrayField(jsl.StringField(), required=False)
     risk_score = jsl.IntField(minimum=0, maximum=100, required=True, default=21)
+    risk_score_mapping = jsl.ArrayField(jsl.DocumentField(RiskScoreMapping), required=False, min_items=1)
     rule_id = jsl.StringField(pattern=UUID_PATTERN, required=True)
+    rule_name_override = jsl.StringField(required=False)
     severity = jsl.StringField(enum=['low', 'medium', 'high', 'critical'], default='low', required=True)
+    severity_mapping = jsl.ArrayField(jsl.DocumentField(SeverityMapping), required=False, min_items=1)
     # saved_id - type must be 'saved_query' to allow this or else it is forbidden
     tags = jsl.ArrayField(jsl.StringField(), required=False)
     throttle = jsl.StringField(required=False)
     timeline_id = jsl.StringField(required=False)
     timeline_title = jsl.StringField(required=False)
+    timestamp_override = jsl.StringField(required=False)
     to = jsl.StringField(required=False, default='now')
     # require this to be always validated with a role
     # type = jsl.StringField(enum=[MACHINE_LEARNING, QUERY, SAVED_QUERY], required=True)

rules/README.md

@@ -8,10 +8,10 @@ Rules within this folder are organized by solution or platform. The structure is
 | [`apm/`](apm)                       | Rules that use Application Performance Monitoring (APM) data sources |
 | [`aws/`](aws)                       | Rules written for the Amazon Web Services (AWS) module of filebeat   |
 | `cross-platform/`                   | Rules that apply to multiple platforms, such as Windows and Linux    |
-| [`endpoint/`](endpoint)             | Rules specifically for Elastic Endpoint Security solution            |
 | [`linux/`](linux)                   | Rules for Linux or other Unix based operating systems                |
 | `macos/`                            | Rules for macOS                                                      |
 | [`ml/`](ml)                         | Rules that use machine learning jobs (ML)                            |
 | [`network/`](network)               | Rules that use network data sources                                  |
 | [`okta/`](okta)                     | Rules written for the Okta module of filebeat                        |
+| [`promotions/`](promotions)         | Rules that promote external alerts into detection engine alerts      |
 | [`windows/`](windows)               | Rules for the Microsoft Windows Operating System                     |

rules/promotions/elastic_endpoint.toml

@@ -0,0 +1,60 @@
+[metadata]
+creation_date = "2020/07/08"
+ecs_version = ["1.5.0"]
+maturity = "production"
+updated_date = "2020/07/08"
+
+[rule]
+author = ["Elastic"]
+description = """
+Generates a detection alert each time an Elastic Endpoint alert is received. Enabling this rule allows you to
+immediately begin investigating your Elastic Endpoint alerts.
+"""
+enabled = true
+from = "now-10m"
+index = ["logs-endpoint.alerts-*"]
+language = "kuery"
+license = "Elastic License"
+max_signals = 10000
+name = "Elastic Endpoint"
+risk_score = 47
+rule_id = "9a1a2dae-0b5f-4c3d-8305-a268d404c306"
+rule_name_override = "message"
+severity = "medium"
+tags = ["Elastic", "Endpoint"]
+timestamp_override = "event.ingested"
+type = "query"
+
+query = '''
+event.kind:alert and event.module:(endpoint and not endgame)
+'''
+
+
+[[rule.risk_score_mapping]]
+field = "event.risk_score"
+operator = "equals"
+value = ""
+
+[[rule.severity_mapping]]
+field = "event.severity"
+operator = "equals"
+value = "21"
+severity = "low"
+
+[[rule.severity_mapping]]
+field = "event.severity"
+operator = "equals"
+value = "47"
+severity = "medium"
+
+[[rule.severity_mapping]]
+field = "event.severity"
+operator = "equals"
+value = "73"
+severity = "high"
+
+[[rule.severity_mapping]]
+field = "event.severity"
+operator = "equals"
+value = "99"
+severity = "critical"

rules/promotions/external_alerts.toml

@@ -0,0 +1,57 @@
+[metadata]
+creation_date = "2020/07/08"
+ecs_version = ["1.5.0"]
+maturity = "production"
+updated_date = "2020/07/08"
+
+[rule]
+author = ["Elastic"]
+description = """
+Generates a detection alert for each external alert written to the configured securitySolution:defaultIndex. Enabling
+this rule allows you to immediately begin investigating external alerts in the app.
+"""
+language = "kuery"
+license = "Elastic License"
+max_signals = 10000
+name = "External Alerts"
+risk_score = 47
+rule_id = "eb079c62-4481-4d6e-9643-3ca499df7aaa"
+rule_name_override = "message"
+severity = "medium"
+tags = ["Elastic"]
+timestamp_override = "event.ingested"
+type = "query"
+
+query = '''
+event.kind:alert and not event.module:(endgame or endpoint)
+'''
+
+
+[[rule.risk_score_mapping]]
+field = "event.risk_score"
+operator = "equals"
+value = ""
+
+[[rule.severity_mapping]]
+field = "event.severity"
+operator = "equals"
+value = "21"
+severity = "low"
+
+[[rule.severity_mapping]]
+field = "event.severity"
+operator = "equals"
+value = "47"
+severity = "medium"
+
+[[rule.severity_mapping]]
+field = "event.severity"
+operator = "equals"
+value = "73"
+severity = "high"
+
+[[rule.severity_mapping]]
+field = "event.severity"
+operator = "equals"
+value = "99"
+severity = "critical"