Adds commands to manage ATT&CK mappings (#2343)

* add att&ck commands; fix 2 rule mappings

* update message to stdout

* updated date for rule changes

* unrelated click bug fix

* add type hinting

detection_rules/attack.py

@@ -7,24 +7,27 @@
 import re
 import time
 from pathlib import Path
+from typing import Optional
 
 import json
 import requests
 from collections import OrderedDict
 
 from .semver import Version
-from .utils import get_etc_path, get_etc_glob_path, read_gzip, gzip_compress
+from .utils import cached, clear_caches, get_etc_path, get_etc_glob_path, read_gzip, gzip_compress
 
 PLATFORMS = ['Windows', 'macOS', 'Linux']
-CROSSWALK_FILE = get_etc_path('attack-crosswalk.json')
-TECHNIQUES_REDIRECT_FILE = get_etc_path('attack-technique-redirects.json')
-
-with open(TECHNIQUES_REDIRECT_FILE, 'r') as f:
-    techniques_redirect_map = json.load(f)['mapping']
+CROSSWALK_FILE = Path(get_etc_path('attack-crosswalk.json'))
+TECHNIQUES_REDIRECT_FILE = Path(get_etc_path('attack-technique-redirects.json'))
 
 tactics_map = {}
 
 
+@cached
+def load_techniques_redirect() -> dict:
+    return json.loads(TECHNIQUES_REDIRECT_FILE.read_text())['mapping']
+
+
 def get_attack_file_path() -> str:
     pattern = 'attack-v*.json.gz'
     attack_file = get_etc_glob_path(pattern)
@@ -91,7 +94,7 @@ technique_id_list = [t for t in technique_lookup if '.' not in t]
 sub_technique_id_list = [t for t in technique_lookup if '.' in t]
 
 
-def refresh_attack_data(save=True):
+def refresh_attack_data(save=True) -> (Optional[dict], Optional[bytes]):
     """Refresh ATT&CK data from Mitre."""
     attack_path = Path(get_attack_file_path())
     filename, _, _ = attack_path.name.rsplit('.', 2)
@@ -111,7 +114,7 @@ def refresh_attack_data(save=True):
 
     if Version(current_version) >= Version(latest_version):
         print(f'No versions newer than the current detected: {current_version}')
-        return
+        return None, None
 
     download = f'https://raw.githubusercontent.com/mitre/cti/{release_name}/enterprise-attack/enterprise-attack.json'
     r = requests.get(download)
@@ -130,6 +133,7 @@ def refresh_attack_data(save=True):
 
 def build_threat_map_entry(tactic: str, *technique_ids: str) -> dict:
     """Build rule threat map from technique IDs."""
+    techniques_redirect_map = load_techniques_redirect()
     url_base = 'https://attack.mitre.org/{type}/{id}/'
     tactic_id = tactics_map[tactic]
     tech_entries = {}
@@ -218,22 +222,19 @@ def build_redirected_techniques_map(threads=50):
     return technique_map
 
 
-def refresh_redirected_techniques_map(threads=50):
+def refresh_redirected_techniques_map(threads: int = 50):
     """Refresh the locally saved copy of the mapping."""
-    global techniques_redirect_map
-
     replacement_map = build_redirected_techniques_map(threads)
     mapping = {'saved_date': time.asctime(), 'mapping': replacement_map}
 
-    with open(TECHNIQUES_REDIRECT_FILE, 'w') as f:
-        json.dump(mapping, f, sort_keys=True, indent=2)
-
-    techniques_redirect_map = mapping
+    TECHNIQUES_REDIRECT_FILE.write_text(json.dumps(mapping, sort_keys=True, indent=2))
+    # reset the cached redirect contents
+    clear_caches()
 
     print(f'refreshed mapping file: {TECHNIQUES_REDIRECT_FILE}')
 
 
-def load_crosswalk_map():
+@cached
+def load_crosswalk_map() -> dict:
     """Retrieve the replacement mapping."""
-    with open(CROSSWALK_FILE, 'r') as f:
-        return json.load(f)['mapping']
+    return json.loads(CROSSWALK_FILE.read_text())['mapping']

detection_rules/devtools.py

@@ -26,8 +26,8 @@ from elasticsearch import Elasticsearch
 from eql.table import Table
 
 from kibana.connector import Kibana
-from . import rule_loader, utils
-from .cli_utils import single_collection
+from . import attack, rule_loader, utils
+from .cli_utils import single_collection, multi_collection
 from .docs import IntegrationSecurityDocs
 from .endgame import EndgameSchemaManager
 from .eswrap import CollectEvents, add_range_to_dsl
@@ -1157,3 +1157,74 @@ def generate_endgame_schema(token: str, endgame_version: str, overwrite: bool):
     client = github.authenticated_client
     schema_manager = EndgameSchemaManager(client, endgame_version)
     schema_manager.save_schemas(overwrite=overwrite)
+
+
+@dev_group.group('attack')
+def attack_group():
+    """Commands for managing Mitre ATT&CK data and mappings."""
+
+
+@attack_group.command('refresh-data')
+def refresh_attack_data() -> dict:
+    """Refresh the ATT&CK data file."""
+    data, _ = attack.refresh_attack_data()
+    return data
+
+
+@attack_group.command('refresh-redirect-mappings')
+def refresh_threat_mappings():
+    """Refresh the ATT&CK redirect file and update all rule threat mappings."""
+    # refresh the attack_technique_redirects
+    click.echo('refreshing data in attack_technique_redirects.json')
+    attack.refresh_redirected_techniques_map()
+
+
+@attack_group.command('update-rules')
+@multi_collection
+def update_attack_in_rules(rules: RuleCollection) -> List[Optional[TOMLRule]]:
+    """Update threat mappings attack data in all rules."""
+    new_rules = []
+    redirected_techniques = attack.load_techniques_redirect()
+    today = time.strftime('%Y/%m/%d')
+
+    for rule in rules.rules:
+        needs_update = False
+        valid_threat: List[ThreatMapping] = []
+        threat_pending_update = {}
+        threat = rule.contents.data.threat or []
+
+        for entry in threat:
+            tactic = entry.tactic.name
+            techniques = []
+            for technique in entry.technique or []:
+                techniques.append(technique.id)
+                techniques.extend([st.id for st in technique.subtechnique or []])
+
+            if any([t for t in techniques if t in redirected_techniques]):
+                needs_update = True
+                threat_pending_update[tactic] = techniques
+            else:
+                valid_threat.append(entry)
+
+        if needs_update:
+            for tactic, techniques in threat_pending_update.items():
+                try:
+                    updated_threat = attack.build_threat_map_entry(tactic, *techniques)
+                except ValueError as err:
+                    raise ValueError(f'{rule.id} - {rule.name}: {err}')
+
+                tm = ThreatMapping.from_dict(updated_threat)
+                valid_threat.append(tm)
+
+            new_meta = dataclasses.replace(rule.contents.metadata, updated_date=today)
+            new_data = dataclasses.replace(rule.contents.data, threat=valid_threat)
+            new_contents = dataclasses.replace(rule.contents, data=new_data, metadata=new_meta)
+            new_rule = TOMLRule(contents=new_contents)
+            new_rule.save_toml()
+            new_rules.append(new_rule)
+
+    if new_rules:
+        click.echo(f'{len(new_rules)} rules updated')
+    else:
+        click.echo('No rule changes needed')
+    return new_rules

detection_rules/eswrap.py

@@ -6,6 +6,7 @@
 """Elasticsearch cli commands."""
 import json
 import os
+import sys
 import time
 from collections import defaultdict
 from typing import List, Union
@@ -347,7 +348,7 @@ def es_group(ctx: click.Context, **kwargs):
     ctx.ensure_object(dict)
 
     # only initialize an es client if the subcommand is invoked without help (hacky)
-    if click.get_os_args()[-1] in ctx.help_option_names:
+    if sys.argv[-1] in ctx.help_option_names:
         click.echo('Elasticsearch client:')
         click.echo(format_command_options(ctx))
 

detection_rules/etc/attack-technique-redirects.json

@@ -19,6 +19,7 @@
     "T1044": "T1574.010",
     "T1045": "T1027.002",
     "T1050": "T1543.003",
+    "T1053.001": "T1053.002",
     "T1054": "T1562.006",
     "T1058": "T1574.011",
     "T1060": "T1547.001",
@@ -128,7 +129,8 @@
     "T1519": "T1546.014",
     "T1522": "T1552.005",
     "T1527": "T1550.001",
-    "T1536": "T1578.004"
+    "T1536": "T1578.004",
+    "T1547.011": "T1647"
   },
-  "saved_date": "Tue Dec  8 23:08:53 2020"
+  "saved_date": "Tue Oct  4 21:58:48 2022"
 }

detection_rules/kbwrap.py

@@ -4,6 +4,7 @@
 # 2.0.
 
 """Kibana cli commands."""
+import sys
 import uuid
 
 import click
@@ -25,7 +26,7 @@ def kibana_group(ctx: click.Context, **kibana_kwargs):
     ctx.ensure_object(dict)
 
     # only initialize an kibana client if the subcommand is invoked without help (hacky)
-    if click.get_os_args()[-1] in ctx.help_option_names:
+    if sys.argv[-1] in ctx.help_option_names:
         click.echo('Kibana client:')
         click.echo(format_command_options(ctx))
 

detection_rules/utils.py

@@ -126,7 +126,7 @@ def save_etc_dump(contents, *path, **kwargs):
         return eql.utils.save_dump(contents, path)
 
 
-def gzip_compress(contents):
+def gzip_compress(contents) -> bytes:
     gz_file = io.BytesIO()
 
     with gzip.GzipFile(mode="w", fileobj=gz_file) as f:

rules/macos/persistence_creation_hidden_login_item_osascript.toml

@@ -3,7 +3,7 @@ creation_date = "2020/01/05"
 maturity = "production"
 min_stack_comments = "New fields added: required_fields, related_integrations, setup"
 min_stack_version = "8.3.0"
-updated_date = "2022/08/24"
+updated_date = "2022/10/07"
 
 [rule]
 author = ["Elastic"]
@@ -35,36 +35,49 @@ process where event.type in ("start", "process_started") and process.name : "osa
 
 [[rule.threat]]
 framework = "MITRE ATT&CK"
-[[rule.threat.technique]]
-id = "T1547"
-name = "Boot or Logon Autostart Execution"
-reference = "https://attack.mitre.org/techniques/T1547/"
-[[rule.threat.technique.subtechnique]]
-id = "T1547.011"
-name = "Plist Modification"
-reference = "https://attack.mitre.org/techniques/T1547/011/"
-
-
 
 [rule.threat.tactic]
 id = "TA0003"
 name = "Persistence"
 reference = "https://attack.mitre.org/tactics/TA0003/"
+
+[[rule.threat.technique]]
+id = "T1547"
+name = "Boot or Logon Autostart Execution"
+reference = "https://attack.mitre.org/techniques/T1547/"
+
+
 [[rule.threat]]
 framework = "MITRE ATT&CK"
-[[rule.threat.technique]]
-id = "T1059"
-name = "Command and Scripting Interpreter"
-reference = "https://attack.mitre.org/techniques/T1059/"
-[[rule.threat.technique.subtechnique]]
-id = "T1059.002"
-name = "AppleScript"
-reference = "https://attack.mitre.org/techniques/T1059/002/"
-
-
 
 [rule.threat.tactic]
 id = "TA0002"
 name = "Execution"
 reference = "https://attack.mitre.org/tactics/TA0002/"
 
+[[rule.threat.technique]]
+id = "T1059"
+name = "Command and Scripting Interpreter"
+reference = "https://attack.mitre.org/techniques/T1059/"
+
+[[rule.threat.technique.subtechnique]]
+id = "T1059.002"
+name = "AppleScript"
+reference = "https://attack.mitre.org/techniques/T1059/002/"
+
+
+[[rule.threat]]
+framework = "MITRE ATT&CK"
+
+[rule.threat.tactic]
+id = "TA0005"
+name = "Defense Evasion"
+reference = "https://attack.mitre.org/tactics/TA0005/"
+
+[[rule.threat.technique]]
+id = "T1647"
+name = "Plist File Modification"
+reference = "https://attack.mitre.org/techniques/T1647/"
+
+
+

rules/macos/persistence_loginwindow_plist_modification.toml

@@ -3,7 +3,7 @@ creation_date = "2021/01/21"
 maturity = "production"
 min_stack_comments = "New fields added: required_fields, related_integrations, setup"
 min_stack_version = "8.3.0"
-updated_date = "2022/08/24"
+updated_date = "2022/10/07"
 
 [rule]
 author = ["Elastic"]
@@ -37,19 +37,28 @@ event.category:"file" and not event.type:"deletion" and
 
 [[rule.threat]]
 framework = "MITRE ATT&CK"
-[[rule.threat.technique]]
-id = "T1547"
-name = "Boot or Logon Autostart Execution"
-reference = "https://attack.mitre.org/techniques/T1547/"
-[[rule.threat.technique.subtechnique]]
-id = "T1547.011"
-name = "Plist Modification"
-reference = "https://attack.mitre.org/techniques/T1547/011/"
-
-
 
 [rule.threat.tactic]
 id = "TA0003"
 name = "Persistence"
 reference = "https://attack.mitre.org/tactics/TA0003/"
 
+[[rule.threat.technique]]
+id = "T1547"
+name = "Boot or Logon Autostart Execution"
+reference = "https://attack.mitre.org/techniques/T1547/"
+
+
+[[rule.threat]]
+framework = "MITRE ATT&CK"
+
+[rule.threat.tactic]
+id = "TA0005"
+name = "Defense Evasion"
+reference = "https://attack.mitre.org/tactics/TA0005/"
+
+[[rule.threat.technique]]
+id = "T1647"
+name = "Plist File Modification"
+reference = "https://attack.mitre.org/techniques/T1647/"
+

tests/test_all_rules.py

@@ -93,7 +93,7 @@ class TestThreatMappings(BaseRuleTest):
 
     def test_technique_deprecations(self):
         """Check for use of any ATT&CK techniques that have been deprecated."""
-        replacement_map = attack.techniques_redirect_map
+        replacement_map = attack.load_techniques_redirect()
         revoked = list(attack.revoked)
         deprecated = list(attack.deprecated)