Ross Wolf
c0af222e7e
* WIP: Convert Rule to a dataclass * Fix make release * Lint fixes * Remove dead code * Fix lint and tests * Use Python 3.8 in GitHub actions * Update README to 3.8+ * Add Python 3.8 assertion * Fix is_dirty property * Remove incorrect pop from contents * Add mixin with from_dict() and to_dict() methods * Bypass validation for deprecated rules * Fix rule_prompt * Fix dict_hash usage * Fix rule_event_search * Switch to definitions.Date * Fix toml-lint command, ignoring 'unneeded defaults' * Moved severity Literal to definitions.Severity * Remove BaseMarshmallowDataclass * Fix lint and tests * Add maturity to metadata for rule prompt loop * Fix typo in devtools * Use rule loader to load single rule in toml-lint * Add Schema hint to __schema method * Add MITREAttackURL definition * Fix is_dirty to compare sha<-->sha * Normalize the autoformatted rule output for API and toml-lint * Make the package hash match * Make the rule object mutable but not rule contents * Restore the rules
72 lines
3.2 KiB
Python
72 lines
3.2 KiB
Python
# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
|
# or more contributor license agreements. Licensed under the Elastic License
|
|
# 2.0; you may not use this file except in compliance with the Elastic License
|
|
# 2.0.
|
|
|
|
"""Test that all rules appropriately match against expected data sets."""
|
|
import copy
|
|
import unittest
|
|
import warnings
|
|
|
|
from detection_rules.rule import KQLRuleData
|
|
from . import get_data_files, get_fp_data_files
|
|
from detection_rules import rule_loader
|
|
from detection_rules.utils import combine_sources, evaluate, load_etc_dump
|
|
|
|
|
|
class TestMappings(unittest.TestCase):
|
|
"""Test that all rules appropriately match against expected data sets."""
|
|
|
|
FP_FILES = get_fp_data_files()
|
|
RULES = rule_loader.load_rules().values()
|
|
|
|
def evaluate(self, documents, rule, expected, msg):
|
|
"""KQL engine to evaluate."""
|
|
filtered = evaluate(rule, documents)
|
|
self.assertEqual(expected, len(filtered), msg)
|
|
return filtered
|
|
|
|
def test_true_positives(self):
|
|
"""Test that expected results return against true positives."""
|
|
mismatched_ecs = []
|
|
mappings = load_etc_dump('rule-mapping.yml')
|
|
|
|
for rule in rule_loader.get_production_rules():
|
|
if isinstance(rule.contents.data, KQLRuleData):
|
|
if rule.id not in mappings:
|
|
continue
|
|
|
|
mapping = mappings[rule.id]
|
|
expected = mapping['count']
|
|
sources = mapping.get('sources')
|
|
rta_file = mapping['rta_name']
|
|
|
|
# ensure sources is defined and not empty; schema allows it to not be set since 'pending' bypasses
|
|
self.assertTrue(sources, 'No sources defined for: {} - {} '.format(rule.id, rule.name))
|
|
msg = 'Expected TP results did not match for: {} - {}'.format(rule.id, rule.name)
|
|
|
|
data_files = [get_data_files('true_positives', rta_file).get(s) for s in sources]
|
|
data_file = combine_sources(*data_files)
|
|
results = self.evaluate(data_file, rule, expected, msg)
|
|
|
|
ecs_versions = set([r.get('ecs', {}).get('version') for r in results])
|
|
rule_ecs = set(rule.metadata.get('ecs_version').copy())
|
|
|
|
if not ecs_versions & rule_ecs:
|
|
msg = '{} - {} ecs_versions ({}) not in source data versions ({})'.format(
|
|
rule.id, rule.name, ', '.join(rule_ecs), ', '.join(ecs_versions))
|
|
mismatched_ecs.append(msg)
|
|
|
|
if mismatched_ecs:
|
|
msg = 'Rules detected with source data from ecs versions not listed within the rule: \n{}'.format(
|
|
'\n'.join(mismatched_ecs))
|
|
warnings.warn(msg)
|
|
|
|
def test_false_positives(self):
|
|
"""Test that expected results return against false positives."""
|
|
for rule in rule_loader.get_production_rules():
|
|
if isinstance(rule.contents.data, KQLRuleData):
|
|
for fp_name, merged_data in get_fp_data_files().items():
|
|
msg = 'Unexpected FP match for: {} - {}, against: {}'.format(rule.id, rule.name, fp_name)
|
|
self.evaluate(copy.deepcopy(merged_data), rule, 0, msg)
|