From c1dd3c57adaf1a4b3318acb34dce1069f28bcb7b Mon Sep 17 00:00:00 2001 From: Justin Ibarra Date: Tue, 1 Nov 2022 11:14:40 -0800 Subject: [PATCH] Adds commands to manage ATT&CK mappings (#2343) * add att&ck commands; fix 2 rule mappings * update message to stdout * updated date for rule changes * unrelated click bug fix * add type hinting --- detection_rules/attack.py | 37 ++++----- detection_rules/devtools.py | 75 ++++++++++++++++++- detection_rules/eswrap.py | 3 +- .../etc/attack-technique-redirects.json | 6 +- detection_rules/kbwrap.py | 3 +- detection_rules/utils.py | 2 +- ..._creation_hidden_login_item_osascript.toml | 55 ++++++++------ ...stence_loginwindow_plist_modification.toml | 31 +++++--- tests/test_all_rules.py | 2 +- 9 files changed, 156 insertions(+), 58 deletions(-) diff --git a/detection_rules/attack.py b/detection_rules/attack.py index cdac1fc07..a87f5448e 100644 --- a/detection_rules/attack.py +++ b/detection_rules/attack.py @@ -7,24 +7,27 @@ import re import time from pathlib import Path +from typing import Optional import json import requests from collections import OrderedDict from .semver import Version -from .utils import get_etc_path, get_etc_glob_path, read_gzip, gzip_compress +from .utils import cached, clear_caches, get_etc_path, get_etc_glob_path, read_gzip, gzip_compress PLATFORMS = ['Windows', 'macOS', 'Linux'] -CROSSWALK_FILE = get_etc_path('attack-crosswalk.json') -TECHNIQUES_REDIRECT_FILE = get_etc_path('attack-technique-redirects.json') - -with open(TECHNIQUES_REDIRECT_FILE, 'r') as f: - techniques_redirect_map = json.load(f)['mapping'] +CROSSWALK_FILE = Path(get_etc_path('attack-crosswalk.json')) +TECHNIQUES_REDIRECT_FILE = Path(get_etc_path('attack-technique-redirects.json')) tactics_map = {} +@cached +def load_techniques_redirect() -> dict: + return json.loads(TECHNIQUES_REDIRECT_FILE.read_text())['mapping'] + + def get_attack_file_path() -> str: pattern = 'attack-v*.json.gz' attack_file = get_etc_glob_path(pattern) @@ -91,7 +94,7 @@ technique_id_list = [t for t in technique_lookup if '.' not in t] sub_technique_id_list = [t for t in technique_lookup if '.' in t] -def refresh_attack_data(save=True): +def refresh_attack_data(save=True) -> (Optional[dict], Optional[bytes]): """Refresh ATT&CK data from Mitre.""" attack_path = Path(get_attack_file_path()) filename, _, _ = attack_path.name.rsplit('.', 2) @@ -111,7 +114,7 @@ def refresh_attack_data(save=True): if Version(current_version) >= Version(latest_version): print(f'No versions newer than the current detected: {current_version}') - return + return None, None download = f'https://raw.githubusercontent.com/mitre/cti/{release_name}/enterprise-attack/enterprise-attack.json' r = requests.get(download) @@ -130,6 +133,7 @@ def refresh_attack_data(save=True): def build_threat_map_entry(tactic: str, *technique_ids: str) -> dict: """Build rule threat map from technique IDs.""" + techniques_redirect_map = load_techniques_redirect() url_base = 'https://attack.mitre.org/{type}/{id}/' tactic_id = tactics_map[tactic] tech_entries = {} @@ -218,22 +222,19 @@ def build_redirected_techniques_map(threads=50): return technique_map -def refresh_redirected_techniques_map(threads=50): +def refresh_redirected_techniques_map(threads: int = 50): """Refresh the locally saved copy of the mapping.""" - global techniques_redirect_map - replacement_map = build_redirected_techniques_map(threads) mapping = {'saved_date': time.asctime(), 'mapping': replacement_map} - with open(TECHNIQUES_REDIRECT_FILE, 'w') as f: - json.dump(mapping, f, sort_keys=True, indent=2) - - techniques_redirect_map = mapping + TECHNIQUES_REDIRECT_FILE.write_text(json.dumps(mapping, sort_keys=True, indent=2)) + # reset the cached redirect contents + clear_caches() print(f'refreshed mapping file: {TECHNIQUES_REDIRECT_FILE}') -def load_crosswalk_map(): +@cached +def load_crosswalk_map() -> dict: """Retrieve the replacement mapping.""" - with open(CROSSWALK_FILE, 'r') as f: - return json.load(f)['mapping'] + return json.loads(CROSSWALK_FILE.read_text())['mapping'] diff --git a/detection_rules/devtools.py b/detection_rules/devtools.py index f6bc27378..b2a71019e 100644 --- a/detection_rules/devtools.py +++ b/detection_rules/devtools.py @@ -26,8 +26,8 @@ from elasticsearch import Elasticsearch from eql.table import Table from kibana.connector import Kibana -from . import rule_loader, utils -from .cli_utils import single_collection +from . import attack, rule_loader, utils +from .cli_utils import single_collection, multi_collection from .docs import IntegrationSecurityDocs from .endgame import EndgameSchemaManager from .eswrap import CollectEvents, add_range_to_dsl @@ -1157,3 +1157,74 @@ def generate_endgame_schema(token: str, endgame_version: str, overwrite: bool): client = github.authenticated_client schema_manager = EndgameSchemaManager(client, endgame_version) schema_manager.save_schemas(overwrite=overwrite) + + +@dev_group.group('attack') +def attack_group(): + """Commands for managing Mitre ATT&CK data and mappings.""" + + +@attack_group.command('refresh-data') +def refresh_attack_data() -> dict: + """Refresh the ATT&CK data file.""" + data, _ = attack.refresh_attack_data() + return data + + +@attack_group.command('refresh-redirect-mappings') +def refresh_threat_mappings(): + """Refresh the ATT&CK redirect file and update all rule threat mappings.""" + # refresh the attack_technique_redirects + click.echo('refreshing data in attack_technique_redirects.json') + attack.refresh_redirected_techniques_map() + + +@attack_group.command('update-rules') +@multi_collection +def update_attack_in_rules(rules: RuleCollection) -> List[Optional[TOMLRule]]: + """Update threat mappings attack data in all rules.""" + new_rules = [] + redirected_techniques = attack.load_techniques_redirect() + today = time.strftime('%Y/%m/%d') + + for rule in rules.rules: + needs_update = False + valid_threat: List[ThreatMapping] = [] + threat_pending_update = {} + threat = rule.contents.data.threat or [] + + for entry in threat: + tactic = entry.tactic.name + techniques = [] + for technique in entry.technique or []: + techniques.append(technique.id) + techniques.extend([st.id for st in technique.subtechnique or []]) + + if any([t for t in techniques if t in redirected_techniques]): + needs_update = True + threat_pending_update[tactic] = techniques + else: + valid_threat.append(entry) + + if needs_update: + for tactic, techniques in threat_pending_update.items(): + try: + updated_threat = attack.build_threat_map_entry(tactic, *techniques) + except ValueError as err: + raise ValueError(f'{rule.id} - {rule.name}: {err}') + + tm = ThreatMapping.from_dict(updated_threat) + valid_threat.append(tm) + + new_meta = dataclasses.replace(rule.contents.metadata, updated_date=today) + new_data = dataclasses.replace(rule.contents.data, threat=valid_threat) + new_contents = dataclasses.replace(rule.contents, data=new_data, metadata=new_meta) + new_rule = TOMLRule(contents=new_contents) + new_rule.save_toml() + new_rules.append(new_rule) + + if new_rules: + click.echo(f'{len(new_rules)} rules updated') + else: + click.echo('No rule changes needed') + return new_rules diff --git a/detection_rules/eswrap.py b/detection_rules/eswrap.py index 5d8bab633..fa056c6b0 100644 --- a/detection_rules/eswrap.py +++ b/detection_rules/eswrap.py @@ -6,6 +6,7 @@ """Elasticsearch cli commands.""" import json import os +import sys import time from collections import defaultdict from typing import List, Union @@ -347,7 +348,7 @@ def es_group(ctx: click.Context, **kwargs): ctx.ensure_object(dict) # only initialize an es client if the subcommand is invoked without help (hacky) - if click.get_os_args()[-1] in ctx.help_option_names: + if sys.argv[-1] in ctx.help_option_names: click.echo('Elasticsearch client:') click.echo(format_command_options(ctx)) diff --git a/detection_rules/etc/attack-technique-redirects.json b/detection_rules/etc/attack-technique-redirects.json index d5dd131ef..5334ba13b 100644 --- a/detection_rules/etc/attack-technique-redirects.json +++ b/detection_rules/etc/attack-technique-redirects.json @@ -19,6 +19,7 @@ "T1044": "T1574.010", "T1045": "T1027.002", "T1050": "T1543.003", + "T1053.001": "T1053.002", "T1054": "T1562.006", "T1058": "T1574.011", "T1060": "T1547.001", @@ -128,7 +129,8 @@ "T1519": "T1546.014", "T1522": "T1552.005", "T1527": "T1550.001", - "T1536": "T1578.004" + "T1536": "T1578.004", + "T1547.011": "T1647" }, - "saved_date": "Tue Dec 8 23:08:53 2020" + "saved_date": "Tue Oct 4 21:58:48 2022" } \ No newline at end of file diff --git a/detection_rules/kbwrap.py b/detection_rules/kbwrap.py index ae1b63bc7..59f11a969 100644 --- a/detection_rules/kbwrap.py +++ b/detection_rules/kbwrap.py @@ -4,6 +4,7 @@ # 2.0. """Kibana cli commands.""" +import sys import uuid import click @@ -25,7 +26,7 @@ def kibana_group(ctx: click.Context, **kibana_kwargs): ctx.ensure_object(dict) # only initialize an kibana client if the subcommand is invoked without help (hacky) - if click.get_os_args()[-1] in ctx.help_option_names: + if sys.argv[-1] in ctx.help_option_names: click.echo('Kibana client:') click.echo(format_command_options(ctx)) diff --git a/detection_rules/utils.py b/detection_rules/utils.py index 5eca054f9..234d64cd0 100644 --- a/detection_rules/utils.py +++ b/detection_rules/utils.py @@ -126,7 +126,7 @@ def save_etc_dump(contents, *path, **kwargs): return eql.utils.save_dump(contents, path) -def gzip_compress(contents): +def gzip_compress(contents) -> bytes: gz_file = io.BytesIO() with gzip.GzipFile(mode="w", fileobj=gz_file) as f: diff --git a/rules/macos/persistence_creation_hidden_login_item_osascript.toml b/rules/macos/persistence_creation_hidden_login_item_osascript.toml index 497a15c41..8b32a0121 100644 --- a/rules/macos/persistence_creation_hidden_login_item_osascript.toml +++ b/rules/macos/persistence_creation_hidden_login_item_osascript.toml @@ -3,7 +3,7 @@ creation_date = "2020/01/05" maturity = "production" min_stack_comments = "New fields added: required_fields, related_integrations, setup" min_stack_version = "8.3.0" -updated_date = "2022/08/24" +updated_date = "2022/10/07" [rule] author = ["Elastic"] @@ -35,36 +35,49 @@ process where event.type in ("start", "process_started") and process.name : "osa [[rule.threat]] framework = "MITRE ATT&CK" -[[rule.threat.technique]] -id = "T1547" -name = "Boot or Logon Autostart Execution" -reference = "https://attack.mitre.org/techniques/T1547/" -[[rule.threat.technique.subtechnique]] -id = "T1547.011" -name = "Plist Modification" -reference = "https://attack.mitre.org/techniques/T1547/011/" - - [rule.threat.tactic] id = "TA0003" name = "Persistence" reference = "https://attack.mitre.org/tactics/TA0003/" + +[[rule.threat.technique]] +id = "T1547" +name = "Boot or Logon Autostart Execution" +reference = "https://attack.mitre.org/techniques/T1547/" + + [[rule.threat]] framework = "MITRE ATT&CK" -[[rule.threat.technique]] -id = "T1059" -name = "Command and Scripting Interpreter" -reference = "https://attack.mitre.org/techniques/T1059/" -[[rule.threat.technique.subtechnique]] -id = "T1059.002" -name = "AppleScript" -reference = "https://attack.mitre.org/techniques/T1059/002/" - - [rule.threat.tactic] id = "TA0002" name = "Execution" reference = "https://attack.mitre.org/tactics/TA0002/" +[[rule.threat.technique]] +id = "T1059" +name = "Command and Scripting Interpreter" +reference = "https://attack.mitre.org/techniques/T1059/" + +[[rule.threat.technique.subtechnique]] +id = "T1059.002" +name = "AppleScript" +reference = "https://attack.mitre.org/techniques/T1059/002/" + + +[[rule.threat]] +framework = "MITRE ATT&CK" + +[rule.threat.tactic] +id = "TA0005" +name = "Defense Evasion" +reference = "https://attack.mitre.org/tactics/TA0005/" + +[[rule.threat.technique]] +id = "T1647" +name = "Plist File Modification" +reference = "https://attack.mitre.org/techniques/T1647/" + + + diff --git a/rules/macos/persistence_loginwindow_plist_modification.toml b/rules/macos/persistence_loginwindow_plist_modification.toml index ad3afc983..9b3cf0e75 100644 --- a/rules/macos/persistence_loginwindow_plist_modification.toml +++ b/rules/macos/persistence_loginwindow_plist_modification.toml @@ -3,7 +3,7 @@ creation_date = "2021/01/21" maturity = "production" min_stack_comments = "New fields added: required_fields, related_integrations, setup" min_stack_version = "8.3.0" -updated_date = "2022/08/24" +updated_date = "2022/10/07" [rule] author = ["Elastic"] @@ -37,19 +37,28 @@ event.category:"file" and not event.type:"deletion" and [[rule.threat]] framework = "MITRE ATT&CK" -[[rule.threat.technique]] -id = "T1547" -name = "Boot or Logon Autostart Execution" -reference = "https://attack.mitre.org/techniques/T1547/" -[[rule.threat.technique.subtechnique]] -id = "T1547.011" -name = "Plist Modification" -reference = "https://attack.mitre.org/techniques/T1547/011/" - - [rule.threat.tactic] id = "TA0003" name = "Persistence" reference = "https://attack.mitre.org/tactics/TA0003/" +[[rule.threat.technique]] +id = "T1547" +name = "Boot or Logon Autostart Execution" +reference = "https://attack.mitre.org/techniques/T1547/" + + +[[rule.threat]] +framework = "MITRE ATT&CK" + +[rule.threat.tactic] +id = "TA0005" +name = "Defense Evasion" +reference = "https://attack.mitre.org/tactics/TA0005/" + +[[rule.threat.technique]] +id = "T1647" +name = "Plist File Modification" +reference = "https://attack.mitre.org/techniques/T1647/" + diff --git a/tests/test_all_rules.py b/tests/test_all_rules.py index 0211be14e..77dd38f6b 100644 --- a/tests/test_all_rules.py +++ b/tests/test_all_rules.py @@ -93,7 +93,7 @@ class TestThreatMappings(BaseRuleTest): def test_technique_deprecations(self): """Check for use of any ATT&CK techniques that have been deprecated.""" - replacement_map = attack.techniques_redirect_map + replacement_map = attack.load_techniques_redirect() revoked = list(attack.revoked) deprecated = list(attack.deprecated)