Add rule loader and dependencies

Co-Authored-By: Justin Ibarra <brokensound77@users.noreply.github.com>
2020-06-29 23:17:38 -06:00
parent a0d3b4bd23
commit 3b305d3003
40 changed files with 138653 additions and 0 deletions
@@ -0,0 +1,238 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+
+"""Definitions for rule metadata and schemas."""
+import time
+
+import jsl
+import jsonschema
+
+from . import ecs
+from .attack import TACTICS, TACTICS_MAP, TECHNIQUES, technique_lookup
+
+UUID_PATTERN = r'[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'
+DATE_PATTERN = r'\d{4}/\d{2}/\d{2}'
+VERSION_PATTERN = r'\d+\.\d+\.\d+'
+RULE_LEVELS = ['recommended', 'aggressive']
+MATURITY_LEVELS = ['development', 'testing', 'staged', 'production', 'deprecated']
+OS_OPTIONS = ['windows', 'linux', 'macos', 'solaris']  # need to verify with ecs
+INTERVAL_PATTERN = r'\d+[mshd]'
+MITRE_URL_PATTERN = r'https://attack.mitre.org/{type}/T[A-Z0-9]+/'
+
+NONFORMATTED_FIELDS = 'note',
+
+
+# kibana/.../siem/server/lib/detection_engine/routes/schemas/add_prepackaged_rules_schema.ts
+#                           /detection_engine/routes/schemas/schemas.ts
+# rule_id is required here
+# output_index is not allowed (and instead the space index must be used)
+# immutable defaults to true instead of to false and if it is there can only be true
+# enabled defaults to false instead of true
+# version is a required field that must exist
+
+MACHINE_LEARNING = 'machine_learning'
+SAVED_QUERY = 'saved_query'
+QUERY = 'query'
+
+
+class FilterMetadata(jsl.Document):
+    """Base class for siem rule meta filters."""
+
+    negate = jsl.BooleanField()
+    type = jsl.StringField()
+    key = jsl.StringField()
+    value = jsl.StringField()
+    disabled = jsl.BooleanField()
+    indexRefName = jsl.StringField()
+    alias = jsl.StringField()  # null acceptable
+    params = jsl.DictField(properties={'query': jsl.StringField()})
+
+
+class FilterQuery(jsl.Document):
+    """Base class for siem rule query filters."""
+
+    match = jsl.DictField({
+        'event.action': jsl.DictField(properties={
+            'query': jsl.StringField(),
+            'type': jsl.StringField()
+        })
+    })
+
+
+class FilterState(jsl.Document):
+    """Base class for siem rule $state filters."""
+
+    store = jsl.StringField()
+
+
+class FilterExists(jsl.Document):
+    """Base class for siem rule $state filters."""
+
+    field = jsl.StringField()
+
+
+class Filters(jsl.Document):
+    """Schema for filters"""
+
+    exists = jsl.DocumentField(FilterExists)
+    meta = jsl.DocumentField(FilterMetadata)
+    state = jsl.DocumentField(FilterState, name='$state')
+    query = jsl.DocumentField(FilterQuery)
+
+
+class ThreatTactic(jsl.Document):
+    """Threat tactics."""
+
+    id = jsl.StringField(enum=TACTICS_MAP.values())
+    name = jsl.StringField(enum=TACTICS)
+    reference = jsl.StringField(MITRE_URL_PATTERN.format(type='tactics'))
+
+
+class ThreatTechnique(jsl.Document):
+    """Threat tactics."""
+
+    id = jsl.StringField(enum=list(technique_lookup))
+    name = jsl.StringField(enum=TECHNIQUES)
+    reference = jsl.StringField(MITRE_URL_PATTERN.format(type='techniques'))
+
+
+class Threat(jsl.Document):
+    """Threat framework mapping such as MITRE ATT&CK."""
+
+    framework = jsl.StringField(default='MITRE ATT&CK', required=True)
+    tactic = jsl.DocumentField(ThreatTactic, required=True)
+    technique = jsl.ArrayField(jsl.DocumentField(ThreatTechnique), required=True)
+
+
+class SiemRuleApiSchema(jsl.Document):
+    """Schema for siem rule in API format."""
+
+    actions = jsl.ArrayField(required=False)
+    author = jsl.ArrayField(jsl.StringField(default="Elastic"), required=True, min_items=1)
+    description = jsl.StringField(required=True)
+    # api defaults to false if blank
+    enabled = jsl.BooleanField(default=False, required=False)
+    exceptions_list = jsl.ArrayField(required=False)
+    # _ required since `from` is a reserved word in python
+    from_ = jsl.StringField(required=False, default='now-6m', name='from')
+    false_positives = jsl.ArrayField(jsl.StringField(), required=False)
+    filters = jsl.ArrayField(jsl.DocumentField(Filters))
+    interval = jsl.StringField(pattern=INTERVAL_PATTERN, default='5m', required=False)
+    license = jsl.StringField(required=True, default="Elastic License")
+    max_signals = jsl.IntField(minimum=1, required=False, default=100)  # cap a max?
+    meta = jsl.DictField(required=False)
+    name = jsl.StringField(required=True)
+    note = jsl.StringField(required=False)
+    # output_index = jsl.StringField(required=False)  # this is NOT allowed!
+    references = jsl.ArrayField(jsl.StringField(), required=False)
+    risk_score = jsl.IntField(minimum=0, maximum=100, required=True, default=21)
+    rule_id = jsl.StringField(pattern=UUID_PATTERN, required=True)
+    severity = jsl.StringField(enum=['low', 'medium', 'high', 'critical'], default='low', required=True)
+    # saved_id - type must be 'saved_query' to allow this or else it is forbidden
+    tags = jsl.ArrayField(jsl.StringField(), required=False)
+    throttle = jsl.StringField(required=False)
+    timeline_id = jsl.StringField(required=False)
+    timeline_title = jsl.StringField(required=False)
+    to = jsl.StringField(required=False, default='now')
+    # require this to be always validated with a role
+    # type = jsl.StringField(enum=[MACHINE_LEARNING, QUERY, SAVED_QUERY], required=True)
+    threat = jsl.ArrayField(jsl.DocumentField(Threat), required=False, min_items=1)
+
+    with jsl.Scope(MACHINE_LEARNING) as ml_scope:
+        ml_scope.anomaly_threshold = jsl.IntField(required=True, minimum=0)
+        ml_scope.machine_learning_job_id = jsl.StringField(required=True)
+        ml_scope.type = jsl.StringField(enum=[MACHINE_LEARNING], required=True, default=MACHINE_LEARNING)
+
+    with jsl.Scope(QUERY) as query_scope:
+        query_scope.index = jsl.ArrayField(jsl.StringField(), required=False)
+        # this is not required per the API but we will enforce it here
+        query_scope.language = jsl.StringField(enum=['kuery', 'lucene'], required=True, default='kuery')
+        query_scope.query = jsl.StringField(required=True)
+        query_scope.type = jsl.StringField(enum=[QUERY], required=True, default=QUERY)
+
+    with jsl.Scope(SAVED_QUERY) as saved_id_scope:
+        saved_id_scope.index = jsl.ArrayField(jsl.StringField(), required=False)
+        saved_id_scope.saved_id = jsl.StringField(required=True)
+        saved_id_scope.type = jsl.StringField(enum=[SAVED_QUERY], required=True, default=SAVED_QUERY)
+
+
+class VersionedApiSchema(SiemRuleApiSchema):
+    """Schema for siem rule in API format with version."""
+
+    version = jsl.IntField(minimum=1, default=1, required=True)
+
+
+class SiemRuleTomlMetadata(jsl.Document):
+    """Schema for siem rule toml metadata."""
+
+    creation_date = jsl.StringField(required=True, pattern=DATE_PATTERN, default=time.strftime('%Y/%m/%d'))
+
+    # added to query with rule.optimize()
+    # rule validated against each ecs schema contained
+    ecs_version = jsl.ArrayField(
+        jsl.StringField(pattern=VERSION_PATTERN, required=True, default=ecs.get_max_version()), required=True)
+    maturity = jsl.StringField(enum=MATURITY_LEVELS, default='development', required=True)
+
+    # if present, add to query
+    os_type_list = jsl.ArrayField(jsl.StringField(enum=OS_OPTIONS), required=False)
+    related_endpoint_rules = jsl.ArrayField(jsl.ArrayField(jsl.StringField(), min_items=2, max_items=2),
+                                            required=False)
+    updated_date = jsl.StringField(required=True, pattern=DATE_PATTERN, default=time.strftime('%Y/%m/%d'))
+
+
+class SiemRuleTomlSchema(jsl.Document):
+    """Schema for siem rule in management toml format."""
+
+    metadata = jsl.DocumentField(SiemRuleTomlMetadata)
+    rule = jsl.DocumentField(SiemRuleApiSchema)
+
+
+class Package(jsl.Document):
+    """Schema for siem rule staging."""
+
+
+class MappingCount(jsl.Document):
+    """Mapping count schema."""
+
+    count = jsl.IntField(minimum=0, required=True)
+    rta_name = jsl.StringField(pattern=r'[a-zA-Z-_]+', required=True)
+    rule_name = jsl.StringField(required=True)
+    sources = jsl.ArrayField(jsl.StringField(), min_items=1)
+
+
+cached_schemas = {}
+
+
+def get_schema(role, as_rule=False, versioned=False):
+    """Get applicable schema by role type and rule format."""
+    if (role, as_rule, versioned) not in cached_schemas:
+        if versioned:
+            cls = VersionedApiSchema
+        else:
+            cls = SiemRuleTomlSchema if as_rule else SiemRuleApiSchema
+
+        cached_schemas[(role, as_rule, versioned)] = cls.get_schema(ordered=True, role=role)
+
+    return cached_schemas[(role, as_rule, versioned)]
+
+
+def schema_validate(contents, as_rule=False, versioned=False):
+    """Validate against all schemas until first hit."""
+    assert isinstance(contents, dict)
+    role = contents.get('rule', {}).get('type') if as_rule else contents.get('type')
+
+    if not role:
+        raise ValueError('Missing rule type!')
+
+    return jsonschema.validate(contents, get_schema(role, as_rule, versioned))
+
+
+metadata_schema = SiemRuleTomlMetadata.get_schema(ordered=True)
+package_schema = Package.get_schema(ordered=True)
+mapping_schema = MappingCount.get_schema(ordered=True)
+
+
+def validate_rta_mapping(mapping):
+    """Validate the RTA mapping."""
+    jsonschema.validate(mapping, mapping_schema)