From 744f56d98e46d5add8042f383eb4af4a200ec31e Mon Sep 17 00:00:00 2001 From: Justin Ibarra Date: Tue, 7 Jun 2022 15:40:46 -0800 Subject: [PATCH] [Bug] resolves bug in Rule version methods (#2021) * [Bug] resolves bug in Rule version methods * comment out unused code with notes --- detection_rules/etc/version.lock.json | 102 ++++++++++++------------- detection_rules/main.py | 4 +- detection_rules/misc.py | 8 +- detection_rules/packaging.py | 8 +- detection_rules/rule.py | 8 +- detection_rules/schemas/__init__.py | 7 +- detection_rules/schemas/definitions.py | 2 + detection_rules/version_lock.py | 92 +++++++++++----------- tests/test_schemas.py | 4 +- tests/test_version_locking.py | 2 +- 10 files changed, 123 insertions(+), 114 deletions(-) diff --git a/detection_rules/etc/version.lock.json b/detection_rules/etc/version.lock.json index 8a35e0419..450f3b5c3 100644 --- a/detection_rules/etc/version.lock.json +++ b/detection_rules/etc/version.lock.json @@ -6,7 +6,7 @@ "version": 6 }, "00140285-b827-4aee-aa09-8113f58a08f3": { - "min_stack_version": "7.13.0", + "min_stack_version": "7.13", "rule_name": "Potential Credential Access via Windows Utilities", "sha256": "6bd8502bc40bd03620c90d9b566806eabce8546ce2a94ee8b2a6afba2bfd8d9a", "type": "eql", @@ -236,7 +236,7 @@ "version": 8 }, "0f93cb9a-1931-48c2-8cd0-f173fd3e5283": { - "min_stack_version": "7.14.0", + "min_stack_version": "7.14", "rule_name": "Potential LSASS Memory Dump via PssCaptureSnapShot", "sha256": "7d16ee5358944e8f1ffcc6a1c546c3bf938b26bcce752e118aaa63d1b5ae3633", "type": "threshold", @@ -327,7 +327,7 @@ "version": 3 }, "138c5dd5-838b-446e-b1ac-c995c7f8108a": { - "min_stack_version": "7.14.0", + "min_stack_version": "7.14", "rule_name": "Rare User Logon", "sha256": "f9e949d45ac4dc51bd454d12b2bd60ec23f8fe3d5ee9a15595a4663248317d73", "type": "machine_learning", @@ -700,9 +700,9 @@ "version": 3 }, "2856446a-34e6-435b-9fb5-f8f040bfa7ed": { - "min_stack_version": "7.16.0", + "min_stack_version": "7.16", "previous": { - "7.16.0": { + "7.16": { "rule_name": "Net command via SYSTEM account", "sha256": "a97a15880fef84d759e6bab118b8f3c882e1cfaa9d51f83415729f840218004a", "type": "eql", @@ -841,7 +841,7 @@ "version": 5 }, "3115bd2c-0baa-4df0-80ea-45e474b5ef93": { - "min_stack_version": "7.15.0", + "min_stack_version": "7.15", "rule_name": "Agent Spoofing - Mismatched Agent ID", "sha256": "d067277b6d08d5e3fe395beecf2eb4a88a5ca6ae5691b52a1d334bae5e23661e", "type": "query", @@ -904,7 +904,7 @@ "34fde489-94b0-4500-a76f-b8a157cf9269": { "min_stack_version": "8.2", "previous": { - "7.16.0": { + "7.16": { "rule_name": "Telnet Port Activity", "sha256": "3dd4a438c915920e6ddb0a5212603af5d94fb8a6b51a32f223d930d7e3becb89", "type": "query", @@ -1091,7 +1091,7 @@ "version": 5 }, "3f0e5410-a4bf-4e8c-bcfc-79d67a285c54": { - "min_stack_version": "7.14.0", + "min_stack_version": "7.14", "rule_name": "CyberArk Privileged Access Security Error", "sha256": "420e91f52a8fb273a099a96a3b3e8beb4c682a608f9ce67d763b32fa803a83dd", "type": "query", @@ -1172,7 +1172,7 @@ "4630d948-40d4-4cef-ac69-4002e29bc3db": { "min_stack_version": "8.2", "previous": { - "7.16.0": { + "7.16": { "rule_name": "Adding Hidden File Attribute via Attrib", "sha256": "0c8c7cbbc5634f75e64baccadab65dea2d7b617c6529b847c00105cadd6b1770", "type": "eql", @@ -1227,7 +1227,7 @@ "version": 1 }, "493834ca-f861-414c-8602-150d5505b777": { - "min_stack_version": "7.15.0", + "min_stack_version": "7.15", "rule_name": "Agent Spoofing - Multiple Hosts Using Same Agent", "sha256": "829bb3432a7664715c5b96c2be6d56e4f957db320f71657203632e61e44b6fe0", "type": "threshold", @@ -1368,7 +1368,7 @@ "54902e45-3467-49a4-8abc-529f2c8cfb80": { "min_stack_version": "8.2", "previous": { - "7.16.0": { + "7.16": { "rule_name": "Uncommon Registry Persistence Change", "sha256": "53219ff8987584e6547f9575812b0376420e95da290d5f3e600c864516a5d0d4", "type": "eql", @@ -1477,9 +1477,9 @@ "version": 4 }, "58c6d58b-a0d3-412d-b3b8-0981a9400607": { - "min_stack_version": "7.16.0", + "min_stack_version": "7.16", "previous": { - "7.16.0": { + "7.16": { "rule_name": "Potential Privilege Escalation via InstallerFileTakeOver", "sha256": "4231315b60c3bf0fa71c1adba0830ae312ed1ab1c6bcec7f91b701ecdd5a1aed", "type": "eql", @@ -1726,7 +1726,7 @@ "version": 3 }, "6839c821-011d-43bd-bd5b-acff00257226": { - "min_stack_version": "7.13.0", + "min_stack_version": "7.13", "rule_name": "Image File Execution Options Injection", "sha256": "6f3da8f7ad3053933ead97d9f24027defb33edf3e295ff028bd18a9028833dda", "type": "eql", @@ -1753,7 +1753,7 @@ "68994a6c-c7ba-4e82-b476-26a26877adf6": { "min_stack_version": "8.0", "previous": { - "7.16.0": { + "7.16": { "rule_name": "Google Workspace Admin Role Assigned to a User", "sha256": "a9e5fed2c237cba481fd05a38576032d3cddf5a3b67341030a4a77725c478b22", "type": "query", @@ -1877,7 +1877,7 @@ "6f435062-b7fc-4af9-acea-5b1ead65c5a5": { "min_stack_version": "8.0", "previous": { - "7.16.0": { + "7.16": { "rule_name": "Google Workspace Role Modified", "sha256": "4776d80c0d1069ed8363242d7b09b4934c3efc58c9db2b87fb5045eda98284e1", "type": "query", @@ -1968,7 +1968,7 @@ "version": 2 }, "745b0119-0560-43ba-860a-7235dd8cee8d": { - "min_stack_version": "7.14.0", + "min_stack_version": "7.14", "rule_name": "Unusual Hour for a User to Logon", "sha256": "cfc6d020a4aff760e43c4f33a76f8e3f56c9aca58b2199c4c498bb3f6f966b42", "type": "machine_learning", @@ -2025,7 +2025,7 @@ "785a404b-75aa-4ffd-8be5-3334a5a544dd": { "min_stack_version": "8.0", "previous": { - "7.16.0": { + "7.16": { "rule_name": "Application Added to Google Workspace Domain", "sha256": "43a87b2b542b409c6cfbe267485d8b1ba8e32e9ea553f6180b7d0362c46ea2d9", "type": "query", @@ -2158,14 +2158,14 @@ "version": 1 }, "850d901a-2a3c-46c6-8b22-55398a01aad8": { - "min_stack_version": "7.15.0", + "min_stack_version": "7.15", "rule_name": "Potential Remote Credential Access via Registry", "sha256": "5c9f1a93f3b025b4be0f335bb2cae5bfc853b437d7f16355b30cd65eabc4520e", "type": "eql", "version": 1 }, "852c1f19-68e8-43a6-9dce-340771fe1be3": { - "min_stack_version": "7.13.0", + "min_stack_version": "7.13", "rule_name": "Suspicious PowerShell Engine ImageLoad", "sha256": "82cc2880a87f37799588a44ac43274cc655633a7c57ff138a6bbd29b7e65b254", "type": "eql", @@ -2280,7 +2280,7 @@ "version": 4 }, "8b2b3a62-a598-4293-bc14-3d5fa22bb98f": { - "min_stack_version": "7.13.0", + "min_stack_version": "7.13", "rule_name": "Executable File Creation with Multiple Extensions", "sha256": "ece6617d0c710bb863cfc4efd2fe61e53bfc9df42a5584c739b063d25a49995a", "type": "eql", @@ -2301,7 +2301,7 @@ "8c1bdde8-4204-45c0-9e0c-c85ca3902488": { "min_stack_version": "8.2", "previous": { - "7.16.0": { + "7.16": { "rule_name": "RDP (Remote Desktop Protocol) from the Internet", "sha256": "b6d7ad4ee2f11ab3ed8aa4bcee08a462a4b3aa3790ae27abd86cee6d921e3283", "type": "query", @@ -2454,7 +2454,7 @@ "93e63c3e-4154-4fc6-9f86-b411e0987bbf": { "min_stack_version": "8.0", "previous": { - "7.16.0": { + "7.16": { "rule_name": "Google Workspace Admin Role Deletion", "sha256": "3c0f93a51365de485043e4961faba1a74302db6036510abbde8f1b0b60e4de3b", "type": "query", @@ -2541,7 +2541,7 @@ "97fc44d3-8dae-4019-ae83-298c3015600f": { "min_stack_version": "8.2", "previous": { - "7.16.0": { + "7.16": { "rule_name": "Startup or Run Key Registry Modification", "sha256": "1827b7a04db141b503dcbe4bdd0c18468ccc43b937e02c76d1f2e7686d2b17ef", "type": "eql", @@ -2590,7 +2590,7 @@ "version": 4 }, "99dcf974-6587-4f65-9252-d866a3fdfd9c": { - "min_stack_version": "7.14.0", + "min_stack_version": "7.14", "rule_name": "Spike in Failed Logon Events", "sha256": "7672fb2df32a9f3da61cb0c2022f18f8bf57af080a3e29e0b647e715d887ef07", "type": "machine_learning", @@ -2623,7 +2623,7 @@ "9c260313-c811-4ec8-ab89-8f6530e0246c": { "min_stack_version": "8.2", "previous": { - "7.16.0": { + "7.16": { "rule_name": "Hosts File Modified", "sha256": "9031db9c1d5f0101bf2e4731e56aaea8eafb32ddeb660da5e3783876162f57d9", "type": "eql", @@ -2768,7 +2768,7 @@ "version": 5 }, "a4c7473a-5cb4-4bc1-9d06-e4a75adbc494": { - "min_stack_version": "7.15.0", + "min_stack_version": "7.15", "rule_name": "Windows Registry File Creation in SMB Share", "sha256": "cc90a0587f15e6896fcc7fcdf8b94c2a6ca43a67d0fcd2a20023a79cc5da21d3", "type": "eql", @@ -2837,7 +2837,7 @@ "a99f82f5-8e77-4f8b-b3ce-10c0f6afbc73": { "min_stack_version": "8.0", "previous": { - "7.16.0": { + "7.16": { "rule_name": "Google Workspace Password Policy Modified", "sha256": "cadc95b5eb7938b3b7310150089830d4dad51e3499916cd2f5c82446659b4051", "type": "query", @@ -2912,7 +2912,7 @@ "acbc8bb9-2486-49a8-8779-45fb5f9a93ee": { "min_stack_version": "8.0", "previous": { - "7.16.0": { + "7.16": { "rule_name": "Google Workspace API Access Granted via Domain-Wide Delegation of Authority", "sha256": "01a8beca2e8f570d63e7614d558243b1d0b9c42d9e0ce9f439b10016f06eaea3", "type": "query", @@ -2957,7 +2957,7 @@ "ad3f2807-2b3e-47d7-b282-f84acbbe14be": { "min_stack_version": "8.0", "previous": { - "7.16.0": { + "7.16": { "rule_name": "Google Workspace Custom Admin Role Created", "sha256": "8b04328630ae74389a2b77d23700d2bfd3900c6008bf0aa9654c2432b427b9c9", "type": "query", @@ -2988,9 +2988,9 @@ "version": 6 }, "afcce5ad-65de-4ed2-8516-5e093d3ac99a": { - "min_stack_version": "7.16.0", + "min_stack_version": "7.16", "previous": { - "7.16.0": { + "7.16": { "rule_name": "Local Scheduled Task Creation", "sha256": "6bef89b0823728244b1f9f53b3bb4cf878d031d22d66d8f1a9ea4ad014ae3537", "type": "eql", @@ -3135,7 +3135,7 @@ "version": 3 }, "b9666521-4742-49ce-9ddc-b8e84c35acae": { - "min_stack_version": "7.13.0", + "min_stack_version": "7.13", "rule_name": "Creation of Hidden Files and Directories", "sha256": "9515b6e94011f55aaec0a81fd8c343771c1bd922a16a699075e105558cb4be3e", "type": "eql", @@ -3358,7 +3358,7 @@ "version": 10 }, "c5f81243-56e0-47f9-b5bb-55a5ed89ba57": { - "min_stack_version": "7.14.0", + "min_stack_version": "7.14", "rule_name": "CyberArk Privileged Access Security Recommended Monitor", "sha256": "0c5ec551b85d7e7e8775c4c1508a831c6019881d679e137e6f0531968d3ab03c", "type": "query", @@ -3475,7 +3475,7 @@ "cad4500a-abd7-4ef3-b5d3-95524de7cfe1": { "min_stack_version": "8.0", "previous": { - "7.16.0": { + "7.16": { "rule_name": "Google Workspace MFA Enforcement Disabled", "sha256": "f8496e8188b47da802b79dba6b01c3f9f4e4d7fe9c0adf98503ec33e0a2f6747", "type": "query", @@ -3574,7 +3574,7 @@ "cf549724-c577-4fd6-8f9b-d1b8ec519ec0": { "min_stack_version": "8.0", "previous": { - "7.16.0": { + "7.16": { "rule_name": "Domain Added to Google Workspace Trusted Domains", "sha256": "5cbeb7ba36d4bca274e78516b67aa418552a39af7ff07d0605a306cacb27a1ef", "type": "query", @@ -3653,7 +3653,7 @@ "version": 2 }, "d4b73fa0-9d43-465e-b8bf-50230da6718b": { - "min_stack_version": "7.14.0", + "min_stack_version": "7.14", "rule_name": "Unusual Source IP for a User to Logon from", "sha256": "eaec6ceda71a7d7f2ef470443ab29248249a5782241bd0d422c6c5201dff280f", "type": "machine_learning", @@ -3728,7 +3728,7 @@ "d76b02ef-fc95-4001-9297-01cb7412232f": { "min_stack_version": "8.2", "previous": { - "7.16.0": { + "7.16": { "rule_name": "Interactive Terminal Spawned via Python", "sha256": "1b8e9ea27c151d2de3fd5c94f0ff8de14098ccc0348a81ac3a39dc28f0dd118f", "type": "query", @@ -3747,7 +3747,7 @@ "version": 1 }, "d7d5c059-c19a-4a96-8ae3-41496ef3bcf9": { - "min_stack_version": "7.14.0", + "min_stack_version": "7.14", "rule_name": "Spike in Logon Events", "sha256": "f597878752cb6e91544579901584b4938249c29026da834e202622b3194aac5b", "type": "machine_learning", @@ -3862,9 +3862,9 @@ "version": 5 }, "e0dacebe-4311-4d50-9387-b17e89c2e7fd": { - "min_stack_version": "7.16.0", + "min_stack_version": "7.16", "previous": { - "7.16.0": { + "7.16": { "rule_name": "Whitespace Padding in Process Command Line", "sha256": "de0b525b55b31026d29a5a835b5e420d95ceaa8d6c6f7e377c3b2cdae2064fdf", "type": "eql", @@ -3901,7 +3901,7 @@ "version": 6 }, "e26aed74-c816-40d3-a810-48d6fbd8b2fd": { - "min_stack_version": "7.14.0", + "min_stack_version": "7.14", "rule_name": "Spike in Logon Events from a Source IP", "sha256": "604e329a73f5f711f4d8aeb944976f58a8d5a993388062231c925fe211be1b91", "type": "machine_learning", @@ -3976,7 +3976,7 @@ "e555105c-ba6d-481f-82bb-9b633e7b4827": { "min_stack_version": "8.0", "previous": { - "7.16.0": { + "7.16": { "rule_name": "MFA Disabled for Google Workspace Organization", "sha256": "1b8f18bfcd5ebd6a7ef2cad523000d799d2cba09cde203a94541c9ad03327c82", "type": "query", @@ -4171,7 +4171,7 @@ "edf8ee23-5ea7-4123-ba19-56b41e424ae3": { "min_stack_version": "8.2", "previous": { - "7.16.0": { + "7.16": { "rule_name": "ImageLoad via Windows Update Auto Update Client", "sha256": "e971abb85880898c0a7f38127565be02a2d427cba85fca159380368553ae06ef", "type": "eql", @@ -4184,9 +4184,9 @@ "version": 7 }, "ee5300a7-7e31-4a72-a258-250abb8b3aa1": { - "min_stack_version": "7.16.0", + "min_stack_version": "7.16", "previous": { - "7.16.0": { + "7.16": { "rule_name": "Unusual Print Spooler Child Process", "sha256": "fe16e0a19a093e954a5c00eb0065d8cb2c1f7064b970bee83ceb761555c259c2", "type": "eql", @@ -4267,7 +4267,7 @@ "f2f46686-6f3c-4724-bd7d-24e31c70f98f": { "min_stack_version": "8.2", "previous": { - "7.16.0": { + "7.16": { "rule_name": "LSASS Memory Dump Creation", "sha256": "c20cf6ad2f9a2341f530aa7cd2335230d2af19bea5f06d81c3d7dbb65e7d38af", "type": "eql", @@ -4394,9 +4394,9 @@ "version": 4 }, "fb02b8d3-71ee-4af1-bacd-215d23f17efa": { - "min_stack_version": "7.16.0", + "min_stack_version": "7.16", "previous": { - "7.16.0": { + "7.16": { "rule_name": "Network Connection via Registration Utility", "sha256": "cdee88e91070d7a8c85aaec9d595418a9392d5e0a0a561789d4a51234aa790c8", "type": "eql", @@ -4441,7 +4441,7 @@ "fd70c98a-c410-42dc-a2e3-761c71848acf": { "min_stack_version": "8.2", "previous": { - "7.16.0": { + "7.16": { "rule_name": "Suspicious CertUtil Commands", "sha256": "3dbede3d16202481d8949fe2200959f78449ea2e1de2ef9d1b2ec9134d16cb35", "type": "eql", @@ -4456,7 +4456,7 @@ "fd7a6052-58fa-4397-93c3-4795249ccfa2": { "min_stack_version": "8.2", "previous": { - "7.16.0": { + "7.16": { "rule_name": "Svchost spawning Cmd", "sha256": "8eda893ef038048202bf4c123453ad33bb5c23dd7808822d6382a5a2361054c8", "type": "eql", diff --git a/detection_rules/main.py b/detection_rules/main.py index 3539bca08..54e8ccbab 100644 --- a/detection_rules/main.py +++ b/detection_rules/main.py @@ -18,7 +18,7 @@ from uuid import uuid4 import click from .cli_utils import rule_prompt, multi_collection -from .misc import add_client, client_error, nested_set, parse_config +from .misc import add_client, client_error, nested_set, parse_config, load_current_package_version from .rule import TOMLRule, TOMLRuleContents from .rule_formatter import toml_write from .rule_loader import RuleCollection @@ -58,7 +58,7 @@ def create_rule(path, config, required_only, rule_type): @click.pass_context def generate_rules_index(ctx: click.Context, query, overwrite, save_files=True): """Generate enriched indexes of rules, based on a KQL search, for indexing/importing into elasticsearch/kibana.""" - from .packaging import load_current_package_version, Package + from .packaging import Package if query: rule_paths = [r['file'] for r in ctx.invoke(search_rules, query=query, verbose=False)] diff --git a/detection_rules/misc.py b/detection_rules/misc.py index b63dd931f..62379e319 100644 --- a/detection_rules/misc.py +++ b/detection_rules/misc.py @@ -29,7 +29,7 @@ except ImportError: GitRelease = None # noqa: N806 GitReleaseAsset = None # noqa: N806 -from .utils import add_params, cached, get_path +from .utils import add_params, cached, get_path, load_etc_dump _CONFIG = {} @@ -249,6 +249,12 @@ def get_kibana_rules(*rule_paths, repo='elastic/kibana', branch='master', verbos return kibana_rules +@cached +def load_current_package_version() -> str: + """Load the current package version from config file.""" + return load_etc_dump('packages.yml')['package']['name'] + + @cached def parse_config(): """Parse a default config file.""" diff --git a/detection_rules/packaging.py b/detection_rules/packaging.py index 790070256..3fdf49010 100644 --- a/detection_rules/packaging.py +++ b/detection_rules/packaging.py @@ -18,7 +18,7 @@ from typing import Dict, Optional, Tuple import click import yaml -from .misc import JS_LICENSE, cached +from .misc import JS_LICENSE, cached, load_current_package_version from .navigator import NavigatorBuilder, Navigator from .rule import TOMLRule, QueryRuleData, ThreatMapping from .rule_loader import DeprecatedCollection, RuleCollection, DEFAULT_RULES_DIR @@ -68,12 +68,6 @@ def filter_rule(rule: TOMLRule, config_filter: dict, exclude_fields: Optional[di return True -@cached -def load_current_package_version() -> str: - """Load the current package version from config file.""" - return load_etc_dump('packages.yml')['package']['name'] - - CURRENT_RELEASE_PATH = Path(RELEASE_DIR) / load_current_package_version() diff --git a/detection_rules/rule.py b/detection_rules/rule.py index c33835f73..471f73d10 100644 --- a/detection_rules/rule.py +++ b/detection_rules/rule.py @@ -21,7 +21,7 @@ import kql from . import utils from .mixins import MarshmallowDataclassMixin from .rule_formatter import toml_write, nested_normalize -from .schemas import SCHEMA_DIR, definitions, downgrade, get_stack_schemas +from .schemas import SCHEMA_DIR, definitions, downgrade, get_stack_schemas, get_min_supported_stack_version from .utils import cached _META_SCHEMA_REQ_DEFAULTS = {} @@ -406,7 +406,8 @@ class BaseRuleContents(ABC): @property def is_dirty(self) -> Optional[bool]: """Determine if the rule has changed since its version was locked.""" - existing_sha256 = self.version_lock.get_locked_hash(self.id, self.metadata.get('min_stack_version')) + min_stack = self.metadata.get('min_stack_version') or str(get_min_supported_stack_version(drop_patch=True)) + existing_sha256 = self.version_lock.get_locked_hash(self.id, min_stack) if existing_sha256 is not None: return existing_sha256 != self.sha256() @@ -414,7 +415,8 @@ class BaseRuleContents(ABC): @property def latest_version(self) -> Optional[int]: """Retrieve the latest known version of the rule.""" - return self.version_lock.get_locked_version(self.id, self.metadata.get('min_stack_version')) + min_stack = self.metadata.get('min_stack_version') or str(get_min_supported_stack_version(drop_patch=True)) + return self.version_lock.get_locked_version(self.id, min_stack) @property def autobumped_version(self) -> Optional[int]: diff --git a/detection_rules/schemas/__init__.py b/detection_rules/schemas/__init__.py index 4fa0b5ac3..51293ec7d 100644 --- a/detection_rules/schemas/__init__.py +++ b/detection_rules/schemas/__init__.py @@ -11,6 +11,7 @@ import jsonschema from . import definitions from .rta_schema import validate_rta_mapping +from ..misc import load_current_package_version from ..semver import Version from ..utils import cached, get_etc_path, load_etc_dump @@ -237,8 +238,6 @@ def load_stack_schema_map() -> dict: @cached def get_stack_schemas(stack_version: Optional[str] = '0.0.0') -> OrderedDictType[str, dict]: """Return all ECS + beats to stack versions for every stack version >= specified stack version and <= package.""" - from ..packaging import load_current_package_version - stack_version = Version(stack_version or '0.0.0') current_package = Version(load_current_package_version()) @@ -269,8 +268,8 @@ def get_stack_versions(drop_patch=False) -> List[str]: return versions -def get_min_supported_stack_version() -> Version: +def get_min_supported_stack_version(drop_patch=False) -> Version: """Get the minimum defined and supported stack version.""" stack_map = load_stack_schema_map() min_version = min(Version(v) for v in list(stack_map)) - return min_version + return Version(min_version[:2]) if drop_patch else min_version diff --git a/detection_rules/schemas/definitions.py b/detection_rules/schemas/definitions.py index 19821e866..766457f08 100644 --- a/detection_rules/schemas/definitions.py +++ b/detection_rules/schemas/definitions.py @@ -24,6 +24,7 @@ UUID_PATTERN = r'^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' _version = r'\d+\.\d+(\.\d+[\w-]*)*' CONDITION_VERSION_PATTERN = rf'^\^{_version}$' VERSION_PATTERN = f'^{_version}$' +MINOR_SEMVER = r'^\d+\.\d+$' BRANCH_PATTERN = f'{VERSION_PATTERN}|^master$' INTERVAL_PATTERN = r'^\d+[mshd]$' @@ -70,6 +71,7 @@ RiskScore = NewType("MaxSignals", int, validate=validate.Range(min=1, max=100)) RuleName = NewType('RuleName', str, validate=validate.Regexp(NAME_PATTERN)) RuleType = Literal['query', 'saved_query', 'machine_learning', 'eql', 'threshold', 'threat_match'] SemVer = NewType('SemVer', str, validate=validate.Regexp(VERSION_PATTERN)) +SemVerMinorOnly = NewType('SemVerFullStrict', str, validate=validate.Regexp(MINOR_SEMVER)) Severity = Literal['low', 'medium', 'high', 'critical'] Sha256 = NewType('Sha256', str, validate=validate.Regexp(SHA256_PATTERN)) SubTechniqueURL = NewType('SubTechniqueURL', str, validate=validate.Regexp(SUBTECHNIQUE_URL)) diff --git a/detection_rules/version_lock.py b/detection_rules/version_lock.py index 58a690e07..27ab6bdb0 100644 --- a/detection_rules/version_lock.py +++ b/detection_rules/version_lock.py @@ -20,7 +20,10 @@ ETC_VERSION_LOCK_FILE = "version.lock.json" ETC_VERSION_LOCK_PATH = Path(get_etc_path()) / ETC_VERSION_LOCK_FILE ETC_DEPRECATED_RULES_FILE = "deprecated_rules.json" ETC_DEPRECATED_RULES_PATH = Path(get_etc_path()) / ETC_DEPRECATED_RULES_FILE -MIN_LOCK_VERSION_DEFAULT = Version("7.13.0") + +# This was the original version the lock was created under. This constant has been replaced by +# schemas.get_min_supported_stack_version to dynamically determine the minimum +# MIN_LOCK_VERSION_DEFAULT = Version("7.13.0") @dataclass(frozen=True) @@ -34,8 +37,8 @@ class BaseEntry: @dataclass(frozen=True) class VersionLockFileEntry(MarshmallowDataclassMixin, BaseEntry): """Schema for a rule entry in the version lock.""" - min_stack_version: Optional[definitions.SemVer] - previous: Optional[Dict[definitions.SemVer, BaseEntry]] + min_stack_version: Optional[definitions.SemVerMinorOnly] + previous: Optional[Dict[definitions.SemVerMinorOnly, BaseEntry]] @dataclass(frozen=True) @@ -82,10 +85,11 @@ class DeprecatedRulesFile(LockDataclassMixin): def _convert_lock_version(stack_version: Optional[str]) -> Version: """Convert an optional stack version to the minimum for the lock.""" - min_version = get_min_supported_stack_version() + min_version = get_min_supported_stack_version(drop_patch=True) if stack_version is None: return min_version - return max(Version(stack_version), min_version) + short_stack_version = Version(Version(stack_version)[:2]) + return max(short_stack_version, min_version) @cached @@ -189,32 +193,26 @@ class VersionLock: return list(changed_rules), list(new_rules), list(newly_deprecated) verbose_echo('Rule changes detected!') - - route = None - existing_rule_lock = {} - original_hash = None changes = [] - def add_changes(r, *msg): - if not original_hash or original_hash != current_rule_lock['sha256']: - new = [f' {route}: {r.id}, new version: {existing_rule_lock["version"]}'] - new.extend([f' - {m}' for m in msg if m]) - changes.extend(new) + def log_changes(r, route_taken, new_rule_version, *msg): + new = [f' {route_taken}: {r.id}, new version: {new_rule_version}'] + new.extend([f' - {m}' for m in msg if m]) + changes.extend(new) for rule in rules: if rule.contents.metadata.maturity == "production" or rule.id in newly_deprecated: # assume that older stacks are always locked first min_stack = _convert_lock_version(rule.contents.metadata.min_stack_version) - current_rule_lock = rule.contents.lock_info(bump=not exclude_version_update) - existing_rule_lock: dict = lock_file_contents.setdefault(rule.id, {}) - original_hash = existing_rule_lock.get('sha256') + lock_from_rule = rule.contents.lock_info(bump=not exclude_version_update) + lock_from_file: dict = lock_file_contents.setdefault(rule.id, {}) # prevent rule type changes for already locked and released rules (#1854) - if existing_rule_lock: - name = current_rule_lock['rule_name'] - existing_type = existing_rule_lock['type'] - current_type = current_rule_lock['type'] + if lock_from_file: + name = lock_from_rule['rule_name'] + existing_type = lock_from_file['type'] + current_type = lock_from_rule['type'] if existing_type != current_type: err_msg = f'cannot change "type" in locked rule: {name} from {existing_type} to {current_type}' raise ValueError(err_msg) @@ -224,41 +222,41 @@ class VersionLock: # 2) on the latest, after a breaking change has been locked # 3) on the latest stack, locking in a breaking change # 4) on an old stack, after a breaking change has been made - latest_locked_stack_version = _convert_lock_version(existing_rule_lock.get("min_stack_version")) + latest_locked_stack_version = _convert_lock_version(lock_from_file.get("min_stack_version")) - if not existing_rule_lock or min_stack == latest_locked_stack_version: + if not lock_from_file or min_stack == latest_locked_stack_version: route = 'A' # 1) no breaking changes ever made or the first time a rule is created # 2) on the latest, after a breaking change has been locked - existing_rule_lock.update(current_rule_lock) + lock_from_file.update(lock_from_rule) + new_version = lock_from_rule['version'] # add the min_stack_version to the lock if it's explicitly set - log_msg = None if rule.contents.metadata.min_stack_version is not None: - existing_rule_lock["min_stack_version"] = str(min_stack) + lock_from_file["min_stack_version"] = str(min_stack) log_msg = f'min_stack_version added: {min_stack}' - - add_changes(rule, log_msg) + log_changes(rule, route, new_version, log_msg) elif min_stack > latest_locked_stack_version: route = 'B' # 3) on the latest stack, locking in a breaking change previous_lock_info = { - "rule_name": existing_rule_lock["rule_name"], - "sha256": existing_rule_lock["sha256"], - "version": existing_rule_lock["version"], - "type": existing_rule_lock["type"] + "rule_name": lock_from_file["rule_name"], + "sha256": lock_from_file["sha256"], + "version": lock_from_file["version"], + "type": lock_from_file["type"] } - existing_rule_lock.setdefault("previous", {}) + lock_from_file.setdefault("previous", {}) # move the current locked info into the previous section - existing_rule_lock["previous"][str(latest_locked_stack_version)] = previous_lock_info + lock_from_file["previous"][str(latest_locked_stack_version)] = previous_lock_info # overwrite the "latest" part of the lock at the top level # TODO: would need to preserve space here as well if supporting forked version spacing - existing_rule_lock.update(current_rule_lock, min_stack_version=str(min_stack)) - add_changes( - rule, + lock_from_file.update(lock_from_rule, min_stack_version=str(min_stack)) + new_version = lock_from_rule['version'] + log_changes( + rule, route, new_version, f'previous {latest_locked_stack_version} saved as version: {previous_lock_info["version"]}', f'current min_stack updated to {min_stack}' ) @@ -266,23 +264,31 @@ class VersionLock: elif min_stack < latest_locked_stack_version: route = 'C' # 4) on an old stack, after a breaking change has been made (updated fork) - assert str(min_stack) in existing_rule_lock.get("previous", {}), \ + assert str(min_stack) in lock_from_file.get("previous", {}), \ f"Expected {rule.id} @ v{min_stack} in the rule lock" # TODO: Figure out whether we support locking old versions and if we want to # "leave room" by skipping versions when breaking changes are made. # We can still inspect the version lock manually after locks are made, # since it's a good summary of everything that happens - existing_rule_lock["previous"][str(min_stack)] = current_rule_lock - existing_rule_lock.update(current_rule_lock) - add_changes(rule, f'previous version {min_stack} updated version to {current_rule_lock["version"]}') + + # if version bump collides with future bump, fail + # if space, change and log + info_from_rule = (lock_from_rule['sha256'], lock_from_rule['version']) + info_from_file = (lock_from_file["previous"][str(min_stack)]['sha256'], + lock_from_file["previous"][str(min_stack)]['version']) + if info_from_rule != info_from_file: + lock_from_file["previous"][str(min_stack)] = lock_from_rule + new_version = lock_from_rule["version"] + log_changes(rule, route, 'unchanged', + f'previous version {min_stack} updated version to {new_version}') continue else: raise RuntimeError("Unreachable code") - if 'previous' in existing_rule_lock: + if 'previous' in lock_from_file: current_rule_version = rule.contents.lock_info()['version'] - for min_stack_version, versioned_lock in existing_rule_lock['previous'].items(): + for min_stack_version, versioned_lock in lock_from_file['previous'].items(): existing_lock_version = versioned_lock['version'] if current_rule_version < existing_lock_version: raise ValueError(f'{rule.id} - previous {min_stack_version=} {existing_lock_version=} ' diff --git a/tests/test_schemas.py b/tests/test_schemas.py index 347fb87de..323f1f0b1 100644 --- a/tests/test_schemas.py +++ b/tests/test_schemas.py @@ -10,7 +10,7 @@ import uuid import eql from detection_rules import utils -from detection_rules.packaging import load_current_package_version +from detection_rules.misc import load_current_package_version from detection_rules.rule import TOMLRuleContents from detection_rules.schemas import downgrade from detection_rules.semver import Version @@ -252,7 +252,7 @@ class TestVersionLockSchema(unittest.TestCase): "34fde489-94b0-4500-a76f-b8a157cf9269": { "min_stack_version": "8.2", "previous": { - "7.13.0": { + "7.13": { "rule_name": "Telnet Port Activity", "sha256": "3dd4a438c915920e6ddb0a5212603af5d94fb8a6b51a32f223d930d7e3becb89", "type": "query", diff --git a/tests/test_version_locking.py b/tests/test_version_locking.py index 9547b1825..2dbd7dd09 100644 --- a/tests/test_version_locking.py +++ b/tests/test_version_locking.py @@ -18,7 +18,7 @@ class TestVersionLock(unittest.TestCase): def test_previous_entries_gte_current_min_stack(self): """Test that all previous entries for all locks in the version lock are >= the current min_stack.""" errors = {} - min_version = get_min_supported_stack_version() + min_version = get_min_supported_stack_version(drop_patch=True) for rule_id, lock in default_version_lock.version_lock.to_dict().items(): if 'previous' in lock: prev_vers = [Version(v) for v in list(lock['previous'])]