diff --git a/detection_rules/devtools.py b/detection_rules/devtools.py index d14588cba..c08f8fb10 100644 --- a/detection_rules/devtools.py +++ b/detection_rules/devtools.py @@ -34,10 +34,10 @@ from .ghwrap import GithubClient, update_gist from .main import root from .misc import PYTHON_LICENSE, add_client, client_error from .packaging import PACKAGE_FILE, RELEASE_DIR, CURRENT_RELEASE_PATH, Package, current_stack_version -from .version_lock import default_version_lock +from .version_lock import VersionLockFile, default_version_lock from .rule import AnyRuleData, BaseRuleData, DeprecatedRule, QueryRuleData, ThreatMapping, TOMLRule from .rule_loader import RuleCollection, production_filter -from .schemas import definitions +from .schemas import definitions, get_stack_versions from .semver import Version from .utils import dict_hash, get_path, load_dump @@ -821,6 +821,46 @@ def update_navigator_gists(directory: Path, token: str, gist_id: str, print_mark return generated_urls +@dev_group.command('trim-version-lock') +@click.argument('min_version') +@click.option('--dry-run', is_flag=True, help='Print the changes rather than saving the file') +def trim_version_lock(min_version: str, dry_run: bool) -> dict: + """Trim all previous entries within the version lock file which are lower than the min_version.""" + min_version = min(Version(v) for v in get_stack_versions(drop_patch=False)) + version_lock_dict = default_version_lock.version_lock.to_dict() + removed = {} + + for rule_id, lock in version_lock_dict.items(): + if 'previous' in lock: + prev_vers = [Version(v) for v in list(lock['previous'])] + outdated_vers = [v for v in prev_vers if v <= min_version] + + if not outdated_vers: + continue + + # we want to remove all "old" versions, but save the latest that is <= the min version as the new + # min_version. Essentially collapsing the entries and bumping it to a new "true" min + latest_version = max(outdated_vers) + + if dry_run: + removed[rule_id] = [str(v) for v in outdated_vers] + for outdated in outdated_vers: + popped = lock['previous'].pop(str(outdated)) + if outdated == latest_version: + lock['previous'][str(min_version)] = popped + + # remove the whole previous entry if it is now blank + if not lock['previous']: + lock.pop('previous') + + if dry_run: + click.echo(f'The following versions would be collapsed to {min_version}:' if removed else 'No changes') + click.echo('\n'.join(f'{k}: {", ".join(v)}' for k, v in removed.items())) + else: + new_lock = VersionLockFile.from_dict(dict(data=version_lock_dict)) + new_lock.save_to_file() + + @dev_group.group('diff') def diff_group(): """Commands for statistics on changes and diffs.""" diff --git a/detection_rules/etc/version.lock.json b/detection_rules/etc/version.lock.json index 7d8cf5050..8a35e0419 100644 --- a/detection_rules/etc/version.lock.json +++ b/detection_rules/etc/version.lock.json @@ -702,7 +702,7 @@ "2856446a-34e6-435b-9fb5-f8f040bfa7ed": { "min_stack_version": "7.16.0", "previous": { - "7.13.0": { + "7.16.0": { "rule_name": "Net command via SYSTEM account", "sha256": "a97a15880fef84d759e6bab118b8f3c882e1cfaa9d51f83415729f840218004a", "type": "eql", @@ -904,7 +904,7 @@ "34fde489-94b0-4500-a76f-b8a157cf9269": { "min_stack_version": "8.2", "previous": { - "7.13.0": { + "7.16.0": { "rule_name": "Telnet Port Activity", "sha256": "3dd4a438c915920e6ddb0a5212603af5d94fb8a6b51a32f223d930d7e3becb89", "type": "query", @@ -1172,7 +1172,7 @@ "4630d948-40d4-4cef-ac69-4002e29bc3db": { "min_stack_version": "8.2", "previous": { - "7.13.0": { + "7.16.0": { "rule_name": "Adding Hidden File Attribute via Attrib", "sha256": "0c8c7cbbc5634f75e64baccadab65dea2d7b617c6529b847c00105cadd6b1770", "type": "eql", @@ -1368,7 +1368,7 @@ "54902e45-3467-49a4-8abc-529f2c8cfb80": { "min_stack_version": "8.2", "previous": { - "7.13.0": { + "7.16.0": { "rule_name": "Uncommon Registry Persistence Change", "sha256": "53219ff8987584e6547f9575812b0376420e95da290d5f3e600c864516a5d0d4", "type": "eql", @@ -1479,7 +1479,7 @@ "58c6d58b-a0d3-412d-b3b8-0981a9400607": { "min_stack_version": "7.16.0", "previous": { - "7.13.0": { + "7.16.0": { "rule_name": "Potential Privilege Escalation via InstallerFileTakeOver", "sha256": "4231315b60c3bf0fa71c1adba0830ae312ed1ab1c6bcec7f91b701ecdd5a1aed", "type": "eql", @@ -1753,7 +1753,7 @@ "68994a6c-c7ba-4e82-b476-26a26877adf6": { "min_stack_version": "8.0", "previous": { - "7.13.0": { + "7.16.0": { "rule_name": "Google Workspace Admin Role Assigned to a User", "sha256": "a9e5fed2c237cba481fd05a38576032d3cddf5a3b67341030a4a77725c478b22", "type": "query", @@ -1877,7 +1877,7 @@ "6f435062-b7fc-4af9-acea-5b1ead65c5a5": { "min_stack_version": "8.0", "previous": { - "7.13.0": { + "7.16.0": { "rule_name": "Google Workspace Role Modified", "sha256": "4776d80c0d1069ed8363242d7b09b4934c3efc58c9db2b87fb5045eda98284e1", "type": "query", @@ -2025,7 +2025,7 @@ "785a404b-75aa-4ffd-8be5-3334a5a544dd": { "min_stack_version": "8.0", "previous": { - "7.13.0": { + "7.16.0": { "rule_name": "Application Added to Google Workspace Domain", "sha256": "43a87b2b542b409c6cfbe267485d8b1ba8e32e9ea553f6180b7d0362c46ea2d9", "type": "query", @@ -2301,7 +2301,7 @@ "8c1bdde8-4204-45c0-9e0c-c85ca3902488": { "min_stack_version": "8.2", "previous": { - "7.13.0": { + "7.16.0": { "rule_name": "RDP (Remote Desktop Protocol) from the Internet", "sha256": "b6d7ad4ee2f11ab3ed8aa4bcee08a462a4b3aa3790ae27abd86cee6d921e3283", "type": "query", @@ -2454,7 +2454,7 @@ "93e63c3e-4154-4fc6-9f86-b411e0987bbf": { "min_stack_version": "8.0", "previous": { - "7.13.0": { + "7.16.0": { "rule_name": "Google Workspace Admin Role Deletion", "sha256": "3c0f93a51365de485043e4961faba1a74302db6036510abbde8f1b0b60e4de3b", "type": "query", @@ -2541,7 +2541,7 @@ "97fc44d3-8dae-4019-ae83-298c3015600f": { "min_stack_version": "8.2", "previous": { - "7.13.0": { + "7.16.0": { "rule_name": "Startup or Run Key Registry Modification", "sha256": "1827b7a04db141b503dcbe4bdd0c18468ccc43b937e02c76d1f2e7686d2b17ef", "type": "eql", @@ -2623,7 +2623,7 @@ "9c260313-c811-4ec8-ab89-8f6530e0246c": { "min_stack_version": "8.2", "previous": { - "7.13.0": { + "7.16.0": { "rule_name": "Hosts File Modified", "sha256": "9031db9c1d5f0101bf2e4731e56aaea8eafb32ddeb660da5e3783876162f57d9", "type": "eql", @@ -2837,7 +2837,7 @@ "a99f82f5-8e77-4f8b-b3ce-10c0f6afbc73": { "min_stack_version": "8.0", "previous": { - "7.13.0": { + "7.16.0": { "rule_name": "Google Workspace Password Policy Modified", "sha256": "cadc95b5eb7938b3b7310150089830d4dad51e3499916cd2f5c82446659b4051", "type": "query", @@ -2912,7 +2912,7 @@ "acbc8bb9-2486-49a8-8779-45fb5f9a93ee": { "min_stack_version": "8.0", "previous": { - "7.13.0": { + "7.16.0": { "rule_name": "Google Workspace API Access Granted via Domain-Wide Delegation of Authority", "sha256": "01a8beca2e8f570d63e7614d558243b1d0b9c42d9e0ce9f439b10016f06eaea3", "type": "query", @@ -2957,7 +2957,7 @@ "ad3f2807-2b3e-47d7-b282-f84acbbe14be": { "min_stack_version": "8.0", "previous": { - "7.13.0": { + "7.16.0": { "rule_name": "Google Workspace Custom Admin Role Created", "sha256": "8b04328630ae74389a2b77d23700d2bfd3900c6008bf0aa9654c2432b427b9c9", "type": "query", @@ -2990,7 +2990,7 @@ "afcce5ad-65de-4ed2-8516-5e093d3ac99a": { "min_stack_version": "7.16.0", "previous": { - "7.13.0": { + "7.16.0": { "rule_name": "Local Scheduled Task Creation", "sha256": "6bef89b0823728244b1f9f53b3bb4cf878d031d22d66d8f1a9ea4ad014ae3537", "type": "eql", @@ -3475,7 +3475,7 @@ "cad4500a-abd7-4ef3-b5d3-95524de7cfe1": { "min_stack_version": "8.0", "previous": { - "7.13.0": { + "7.16.0": { "rule_name": "Google Workspace MFA Enforcement Disabled", "sha256": "f8496e8188b47da802b79dba6b01c3f9f4e4d7fe9c0adf98503ec33e0a2f6747", "type": "query", @@ -3574,7 +3574,7 @@ "cf549724-c577-4fd6-8f9b-d1b8ec519ec0": { "min_stack_version": "8.0", "previous": { - "7.13.0": { + "7.16.0": { "rule_name": "Domain Added to Google Workspace Trusted Domains", "sha256": "5cbeb7ba36d4bca274e78516b67aa418552a39af7ff07d0605a306cacb27a1ef", "type": "query", @@ -3728,7 +3728,7 @@ "d76b02ef-fc95-4001-9297-01cb7412232f": { "min_stack_version": "8.2", "previous": { - "7.13.0": { + "7.16.0": { "rule_name": "Interactive Terminal Spawned via Python", "sha256": "1b8e9ea27c151d2de3fd5c94f0ff8de14098ccc0348a81ac3a39dc28f0dd118f", "type": "query", @@ -3864,7 +3864,7 @@ "e0dacebe-4311-4d50-9387-b17e89c2e7fd": { "min_stack_version": "7.16.0", "previous": { - "7.13.0": { + "7.16.0": { "rule_name": "Whitespace Padding in Process Command Line", "sha256": "de0b525b55b31026d29a5a835b5e420d95ceaa8d6c6f7e377c3b2cdae2064fdf", "type": "eql", @@ -3976,7 +3976,7 @@ "e555105c-ba6d-481f-82bb-9b633e7b4827": { "min_stack_version": "8.0", "previous": { - "7.13.0": { + "7.16.0": { "rule_name": "MFA Disabled for Google Workspace Organization", "sha256": "1b8f18bfcd5ebd6a7ef2cad523000d799d2cba09cde203a94541c9ad03327c82", "type": "query", @@ -4171,7 +4171,7 @@ "edf8ee23-5ea7-4123-ba19-56b41e424ae3": { "min_stack_version": "8.2", "previous": { - "7.13.0": { + "7.16.0": { "rule_name": "ImageLoad via Windows Update Auto Update Client", "sha256": "e971abb85880898c0a7f38127565be02a2d427cba85fca159380368553ae06ef", "type": "eql", @@ -4186,7 +4186,7 @@ "ee5300a7-7e31-4a72-a258-250abb8b3aa1": { "min_stack_version": "7.16.0", "previous": { - "7.13.0": { + "7.16.0": { "rule_name": "Unusual Print Spooler Child Process", "sha256": "fe16e0a19a093e954a5c00eb0065d8cb2c1f7064b970bee83ceb761555c259c2", "type": "eql", @@ -4267,7 +4267,7 @@ "f2f46686-6f3c-4724-bd7d-24e31c70f98f": { "min_stack_version": "8.2", "previous": { - "7.13.0": { + "7.16.0": { "rule_name": "LSASS Memory Dump Creation", "sha256": "c20cf6ad2f9a2341f530aa7cd2335230d2af19bea5f06d81c3d7dbb65e7d38af", "type": "eql", @@ -4396,7 +4396,7 @@ "fb02b8d3-71ee-4af1-bacd-215d23f17efa": { "min_stack_version": "7.16.0", "previous": { - "7.13.0": { + "7.16.0": { "rule_name": "Network Connection via Registration Utility", "sha256": "cdee88e91070d7a8c85aaec9d595418a9392d5e0a0a561789d4a51234aa790c8", "type": "eql", @@ -4441,7 +4441,7 @@ "fd70c98a-c410-42dc-a2e3-761c71848acf": { "min_stack_version": "8.2", "previous": { - "7.13.0": { + "7.16.0": { "rule_name": "Suspicious CertUtil Commands", "sha256": "3dbede3d16202481d8949fe2200959f78449ea2e1de2ef9d1b2ec9134d16cb35", "type": "eql", @@ -4456,7 +4456,7 @@ "fd7a6052-58fa-4397-93c3-4795249ccfa2": { "min_stack_version": "8.2", "previous": { - "7.13.0": { + "7.16.0": { "rule_name": "Svchost spawning Cmd", "sha256": "8eda893ef038048202bf4c123453ad33bb5c23dd7808822d6382a5a2361054c8", "type": "eql", diff --git a/detection_rules/mixins.py b/detection_rules/mixins.py index 6a26fbc35..d00820db6 100644 --- a/detection_rules/mixins.py +++ b/detection_rules/mixins.py @@ -163,9 +163,10 @@ class LockDataclassMixin: contents = self.to_dict() return dict_hash(contents) - def save_to_file(self, lock_file: Path): + def save_to_file(self, lock_file: Optional[Path] = None): """Save and validate a version lock file.""" - path: Path = getattr(self, 'file_path', lock_file) + path: Path = lock_file or getattr(self, 'file_path', None) + assert path, 'No path passed or set' contents = self.to_dict() path.write_text(json.dumps(contents, indent=2, sort_keys=True)) diff --git a/tests/test_gh_workflows.py b/tests/test_gh_workflows.py index fa2371200..076bcb9b2 100644 --- a/tests/test_gh_workflows.py +++ b/tests/test_gh_workflows.py @@ -12,7 +12,6 @@ import yaml from detection_rules.schemas import get_stack_versions from detection_rules.utils import get_path -from detection_rules.packaging import current_stack_version GITHUB_FILES = Path(get_path()) / '.github' GITHUB_WORKFLOWS = GITHUB_FILES / 'workflows' @@ -28,7 +27,5 @@ class TestWorkflows(unittest.TestCase): lock_versions = lock_workflow[True]['workflow_dispatch']['inputs']['branches']['default'].split(',') matrix_versions = get_stack_versions() - # drop the current package since that should not be backported to (since it is main) - matrix_versions.remove(current_stack_version()) err_msg = 'lock-versions workflow default does not match current matrix in stack-schema-map' - self.assertListEqual(lock_versions, matrix_versions, err_msg) + self.assertListEqual(lock_versions, matrix_versions[:-1], err_msg) diff --git a/tests/test_version_locking.py b/tests/test_version_locking.py new file mode 100644 index 000000000..148762b9b --- /dev/null +++ b/tests/test_version_locking.py @@ -0,0 +1,35 @@ +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0; you may not use this file except in compliance with the Elastic License +# 2.0. + +"""Test version locking of rules.""" + +import unittest + +from detection_rules.schemas import get_stack_versions +from detection_rules.semver import Version +from detection_rules.version_lock import default_version_lock + + +class TestVersionLock(unittest.TestCase): + """Test version locking.""" + + def test_previous_entries_gte_current_min_stack(self): + """Test that all previous entries for all locks in the version lock are >= the current min_stack.""" + errors = {} + min_version = min(Version(v) for v in get_stack_versions(drop_patch=False)) + for rule_id, lock in default_version_lock.version_lock.to_dict().items(): + if 'previous' in lock: + prev_vers = [Version(v) for v in list(lock['previous'])] + outdated = [str(v) for v in prev_vers if v < min_version] + if outdated: + errors[rule_id] = outdated + + # This should only ever happen when bumping the backport matrix support up, which is based on the + # stack-schema-map + if errors: + err_str = '\n'.join(f'{k}: {", ".join(v)}' for k, v in errors.items()) + self.fail(f'The following version.lock entries have previous locked versions which are lower than the ' + f'currently supported min_stack ({min_version}). To address this, run the ' + f'`dev trim-version-lock {min_version}` command.\n\n{err_str}')