security-tools/sigma-rules
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Collapse unsupported previous version entries (#2013)

Browse Source 
* Collapse unsupported previous version entries
* drop the last entry in the matrix test

(cherry picked from commit f57950a3c9)
This commit is contained in:
Justin Ibarra
2022-06-02 15:18:12 -08:00
committed by github-actions[bot]
parent 220996b1b8
commit 3a1a5fe12b
5 changed files with 108 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
detection_rules/devtools.py
+42 -2
View File
@@ -34,10 +34,10 @@ from .ghwrap import GithubClient, update_gist
from .main import root
from .misc import PYTHON_LICENSE, add_client, client_error
from .packaging import PACKAGE_FILE, RELEASE_DIR, CURRENT_RELEASE_PATH, Package, current_stack_version
from .version_lock import default_version_lock
from .version_lock import VersionLockFile, default_version_lock
from .rule import AnyRuleData, BaseRuleData, DeprecatedRule, QueryRuleData, ThreatMapping, TOMLRule
from .rule_loader import RuleCollection, production_filter
from .schemas import definitions
from .schemas import definitions, get_stack_versions
from .semver import Version
from .utils import dict_hash, get_path, load_dump
@@ -821,6 +821,46 @@ def update_navigator_gists(directory: Path, token: str, gist_id: str, print_mark
return generated_urls
@dev_group.command('trim-version-lock')
@click.argument('min_version')
@click.option('--dry-run', is_flag=True, help='Print the changes rather than saving the file')
def trim_version_lock(min_version: str, dry_run: bool) -> dict:
"""Trim all previous entries within the version lock file which are lower than the min_version."""
min_version = min(Version(v) for v in get_stack_versions(drop_patch=False))
version_lock_dict = default_version_lock.version_lock.to_dict()
removed = {}
for rule_id, lock in version_lock_dict.items():
if 'previous' in lock:
prev_vers = [Version(v) for v in list(lock['previous'])]
outdated_vers = [v for v in prev_vers if v <= min_version]
if not outdated_vers:
continue
# we want to remove all "old" versions, but save the latest that is <= the min version as the new
# min_version. Essentially collapsing the entries and bumping it to a new "true" min
latest_version = max(outdated_vers)
if dry_run:
removed[rule_id] = [str(v) for v in outdated_vers]
for outdated in outdated_vers:
popped = lock['previous'].pop(str(outdated))
if outdated == latest_version:
lock['previous'][str(min_version)] = popped
# remove the whole previous entry if it is now blank
if not lock['previous']:
lock.pop('previous')
if dry_run:
click.echo(f'The following versions would be collapsed to {min_version}:' if removed else 'No changes')
click.echo('\n'.join(f'{k}: {", ".join(v)}' for k, v in removed.items()))
else:
new_lock = VersionLockFile.from_dict(dict(data=version_lock_dict))
new_lock.save_to_file()
@dev_group.group('diff')
def diff_group():
"""Commands for statistics on changes and diffs."""
detection_rules/etc/version.lock.json
+27 -27
View File
@@ -702,7 +702,7 @@
"2856446a-34e6-435b-9fb5-f8f040bfa7ed": {
"min_stack_version": "7.16.0",
"previous": {
"7.13.0": {
"7.16.0": {
"rule_name": "Net command via SYSTEM account",
"sha256": "a97a15880fef84d759e6bab118b8f3c882e1cfaa9d51f83415729f840218004a",
"type": "eql",
@@ -904,7 +904,7 @@
"34fde489-94b0-4500-a76f-b8a157cf9269": {
"min_stack_version": "8.2",
"previous": {
"7.13.0": {
"7.16.0": {
"rule_name": "Telnet Port Activity",
"sha256": "3dd4a438c915920e6ddb0a5212603af5d94fb8a6b51a32f223d930d7e3becb89",
"type": "query",
@@ -1172,7 +1172,7 @@
"4630d948-40d4-4cef-ac69-4002e29bc3db": {
"min_stack_version": "8.2",
"previous": {
"7.13.0": {
"7.16.0": {
"rule_name": "Adding Hidden File Attribute via Attrib",
"sha256": "0c8c7cbbc5634f75e64baccadab65dea2d7b617c6529b847c00105cadd6b1770",
"type": "eql",
@@ -1368,7 +1368,7 @@
"54902e45-3467-49a4-8abc-529f2c8cfb80": {
"min_stack_version": "8.2",
"previous": {
"7.13.0": {
"7.16.0": {
"rule_name": "Uncommon Registry Persistence Change",
"sha256": "53219ff8987584e6547f9575812b0376420e95da290d5f3e600c864516a5d0d4",
"type": "eql",
@@ -1479,7 +1479,7 @@
"58c6d58b-a0d3-412d-b3b8-0981a9400607": {
"min_stack_version": "7.16.0",
"previous": {
"7.13.0": {
"7.16.0": {
"rule_name": "Potential Privilege Escalation via InstallerFileTakeOver",
"sha256": "4231315b60c3bf0fa71c1adba0830ae312ed1ab1c6bcec7f91b701ecdd5a1aed",
"type": "eql",
@@ -1753,7 +1753,7 @@
"68994a6c-c7ba-4e82-b476-26a26877adf6": {
"min_stack_version": "8.0",
"previous": {
"7.13.0": {
"7.16.0": {
"rule_name": "Google Workspace Admin Role Assigned to a User",
"sha256": "a9e5fed2c237cba481fd05a38576032d3cddf5a3b67341030a4a77725c478b22",
"type": "query",
@@ -1877,7 +1877,7 @@
"6f435062-b7fc-4af9-acea-5b1ead65c5a5": {
"min_stack_version": "8.0",
"previous": {
"7.13.0": {
"7.16.0": {
"rule_name": "Google Workspace Role Modified",
"sha256": "4776d80c0d1069ed8363242d7b09b4934c3efc58c9db2b87fb5045eda98284e1",
"type": "query",
@@ -2025,7 +2025,7 @@
"785a404b-75aa-4ffd-8be5-3334a5a544dd": {
"min_stack_version": "8.0",
"previous": {
"7.13.0": {
"7.16.0": {
"rule_name": "Application Added to Google Workspace Domain",
"sha256": "43a87b2b542b409c6cfbe267485d8b1ba8e32e9ea553f6180b7d0362c46ea2d9",
"type": "query",
@@ -2301,7 +2301,7 @@
"8c1bdde8-4204-45c0-9e0c-c85ca3902488": {
"min_stack_version": "8.2",
"previous": {
"7.13.0": {
"7.16.0": {
"rule_name": "RDP (Remote Desktop Protocol) from the Internet",
"sha256": "b6d7ad4ee2f11ab3ed8aa4bcee08a462a4b3aa3790ae27abd86cee6d921e3283",
"type": "query",
@@ -2454,7 +2454,7 @@
"93e63c3e-4154-4fc6-9f86-b411e0987bbf": {
"min_stack_version": "8.0",
"previous": {
"7.13.0": {
"7.16.0": {
"rule_name": "Google Workspace Admin Role Deletion",
"sha256": "3c0f93a51365de485043e4961faba1a74302db6036510abbde8f1b0b60e4de3b",
"type": "query",
@@ -2541,7 +2541,7 @@
"97fc44d3-8dae-4019-ae83-298c3015600f": {
"min_stack_version": "8.2",
"previous": {
"7.13.0": {
"7.16.0": {
"rule_name": "Startup or Run Key Registry Modification",
"sha256": "1827b7a04db141b503dcbe4bdd0c18468ccc43b937e02c76d1f2e7686d2b17ef",
"type": "eql",
@@ -2623,7 +2623,7 @@
"9c260313-c811-4ec8-ab89-8f6530e0246c": {
"min_stack_version": "8.2",
"previous": {
"7.13.0": {
"7.16.0": {
"rule_name": "Hosts File Modified",
"sha256": "9031db9c1d5f0101bf2e4731e56aaea8eafb32ddeb660da5e3783876162f57d9",
"type": "eql",
@@ -2837,7 +2837,7 @@
"a99f82f5-8e77-4f8b-b3ce-10c0f6afbc73": {
"min_stack_version": "8.0",
"previous": {
"7.13.0": {
"7.16.0": {
"rule_name": "Google Workspace Password Policy Modified",
"sha256": "cadc95b5eb7938b3b7310150089830d4dad51e3499916cd2f5c82446659b4051",
"type": "query",
@@ -2912,7 +2912,7 @@
"acbc8bb9-2486-49a8-8779-45fb5f9a93ee": {
"min_stack_version": "8.0",
"previous": {
"7.13.0": {
"7.16.0": {
"rule_name": "Google Workspace API Access Granted via Domain-Wide Delegation of Authority",
"sha256": "01a8beca2e8f570d63e7614d558243b1d0b9c42d9e0ce9f439b10016f06eaea3",
"type": "query",
@@ -2957,7 +2957,7 @@
"ad3f2807-2b3e-47d7-b282-f84acbbe14be": {
"min_stack_version": "8.0",
"previous": {
"7.13.0": {
"7.16.0": {
"rule_name": "Google Workspace Custom Admin Role Created",
"sha256": "8b04328630ae74389a2b77d23700d2bfd3900c6008bf0aa9654c2432b427b9c9",
"type": "query",
@@ -2990,7 +2990,7 @@
"afcce5ad-65de-4ed2-8516-5e093d3ac99a": {
"min_stack_version": "7.16.0",
"previous": {
"7.13.0": {
"7.16.0": {
"rule_name": "Local Scheduled Task Creation",
"sha256": "6bef89b0823728244b1f9f53b3bb4cf878d031d22d66d8f1a9ea4ad014ae3537",
"type": "eql",
@@ -3475,7 +3475,7 @@
"cad4500a-abd7-4ef3-b5d3-95524de7cfe1": {
"min_stack_version": "8.0",
"previous": {
"7.13.0": {
"7.16.0": {
"rule_name": "Google Workspace MFA Enforcement Disabled",
"sha256": "f8496e8188b47da802b79dba6b01c3f9f4e4d7fe9c0adf98503ec33e0a2f6747",
"type": "query",
@@ -3574,7 +3574,7 @@
"cf549724-c577-4fd6-8f9b-d1b8ec519ec0": {
"min_stack_version": "8.0",
"previous": {
"7.13.0": {
"7.16.0": {
"rule_name": "Domain Added to Google Workspace Trusted Domains",
"sha256": "5cbeb7ba36d4bca274e78516b67aa418552a39af7ff07d0605a306cacb27a1ef",
"type": "query",
@@ -3728,7 +3728,7 @@
"d76b02ef-fc95-4001-9297-01cb7412232f": {
"min_stack_version": "8.2",
"previous": {
"7.13.0": {
"7.16.0": {
"rule_name": "Interactive Terminal Spawned via Python",
"sha256": "1b8e9ea27c151d2de3fd5c94f0ff8de14098ccc0348a81ac3a39dc28f0dd118f",
"type": "query",
@@ -3864,7 +3864,7 @@
"e0dacebe-4311-4d50-9387-b17e89c2e7fd": {
"min_stack_version": "7.16.0",
"previous": {
"7.13.0": {
"7.16.0": {
"rule_name": "Whitespace Padding in Process Command Line",
"sha256": "de0b525b55b31026d29a5a835b5e420d95ceaa8d6c6f7e377c3b2cdae2064fdf",
"type": "eql",
@@ -3976,7 +3976,7 @@
"e555105c-ba6d-481f-82bb-9b633e7b4827": {
"min_stack_version": "8.0",
"previous": {
"7.13.0": {
"7.16.0": {
"rule_name": "MFA Disabled for Google Workspace Organization",
"sha256": "1b8f18bfcd5ebd6a7ef2cad523000d799d2cba09cde203a94541c9ad03327c82",
"type": "query",
@@ -4171,7 +4171,7 @@
"edf8ee23-5ea7-4123-ba19-56b41e424ae3": {
"min_stack_version": "8.2",
"previous": {
"7.13.0": {
"7.16.0": {
"rule_name": "ImageLoad via Windows Update Auto Update Client",
"sha256": "e971abb85880898c0a7f38127565be02a2d427cba85fca159380368553ae06ef",
"type": "eql",
@@ -4186,7 +4186,7 @@
"ee5300a7-7e31-4a72-a258-250abb8b3aa1": {
"min_stack_version": "7.16.0",
"previous": {
"7.13.0": {
"7.16.0": {
"rule_name": "Unusual Print Spooler Child Process",
"sha256": "fe16e0a19a093e954a5c00eb0065d8cb2c1f7064b970bee83ceb761555c259c2",
"type": "eql",
@@ -4267,7 +4267,7 @@
"f2f46686-6f3c-4724-bd7d-24e31c70f98f": {
"min_stack_version": "8.2",
"previous": {
"7.13.0": {
"7.16.0": {
"rule_name": "LSASS Memory Dump Creation",
"sha256": "c20cf6ad2f9a2341f530aa7cd2335230d2af19bea5f06d81c3d7dbb65e7d38af",
"type": "eql",
@@ -4396,7 +4396,7 @@
"fb02b8d3-71ee-4af1-bacd-215d23f17efa": {
"min_stack_version": "7.16.0",
"previous": {
"7.13.0": {
"7.16.0": {
"rule_name": "Network Connection via Registration Utility",
"sha256": "cdee88e91070d7a8c85aaec9d595418a9392d5e0a0a561789d4a51234aa790c8",
"type": "eql",
@@ -4441,7 +4441,7 @@
"fd70c98a-c410-42dc-a2e3-761c71848acf": {
"min_stack_version": "8.2",
"previous": {
"7.13.0": {
"7.16.0": {
"rule_name": "Suspicious CertUtil Commands",
"sha256": "3dbede3d16202481d8949fe2200959f78449ea2e1de2ef9d1b2ec9134d16cb35",
"type": "eql",
@@ -4456,7 +4456,7 @@
"fd7a6052-58fa-4397-93c3-4795249ccfa2": {
"min_stack_version": "8.2",
"previous": {
"7.13.0": {
"7.16.0": {
"rule_name": "Svchost spawning Cmd",
"sha256": "8eda893ef038048202bf4c123453ad33bb5c23dd7808822d6382a5a2361054c8",
"type": "eql",
detection_rules/mixins.py
+3 -2
View File
@@ -163,9 +163,10 @@ class LockDataclassMixin:
contents = self.to_dict()
return dict_hash(contents)
def save_to_file(self, lock_file: Path):
def save_to_file(self, lock_file: Optional[Path] = None):
"""Save and validate a version lock file."""
path: Path = getattr(self, 'file_path', lock_file)
path: Path = lock_file or getattr(self, 'file_path', None)
assert path, 'No path passed or set'
contents = self.to_dict()
path.write_text(json.dumps(contents, indent=2, sort_keys=True))
tests/test_gh_workflows.py
+1 -4
View File
@@ -12,7 +12,6 @@ import yaml
from detection_rules.schemas import get_stack_versions
from detection_rules.utils import get_path
from detection_rules.packaging import current_stack_version
GITHUB_FILES = Path(get_path()) / '.github'
GITHUB_WORKFLOWS = GITHUB_FILES / 'workflows'
@@ -28,7 +27,5 @@ class TestWorkflows(unittest.TestCase):
lock_versions = lock_workflow[True]['workflow_dispatch']['inputs']['branches']['default'].split(',')
matrix_versions = get_stack_versions()
# drop the current package since that should not be backported to (since it is main)
matrix_versions.remove(current_stack_version())
err_msg = 'lock-versions workflow default does not match current matrix in stack-schema-map'
self.assertListEqual(lock_versions, matrix_versions, err_msg)
self.assertListEqual(lock_versions, matrix_versions[:-1], err_msg)
tests/test_version_locking.py
+35
View File
@@ -0,0 +1,35 @@
# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
# or more contributor license agreements. Licensed under the Elastic License
# 2.0; you may not use this file except in compliance with the Elastic License
# 2.0.
"""Test version locking of rules."""
import unittest
from detection_rules.schemas import get_stack_versions
from detection_rules.semver import Version
from detection_rules.version_lock import default_version_lock
class TestVersionLock(unittest.TestCase):
"""Test version locking."""
def test_previous_entries_gte_current_min_stack(self):
"""Test that all previous entries for all locks in the version lock are >= the current min_stack."""
errors = {}
min_version = min(Version(v) for v in get_stack_versions(drop_patch=False))
for rule_id, lock in default_version_lock.version_lock.to_dict().items():
if 'previous' in lock:
prev_vers = [Version(v) for v in list(lock['previous'])]
outdated = [str(v) for v in prev_vers if v < min_version]
if outdated:
errors[rule_id] = outdated
# This should only ever happen when bumping the backport matrix support up, which is based on the
# stack-schema-map
if errors:
err_str = '\n'.join(f'{k}: {", ".join(v)}' for k, v in errors.items())
self.fail(f'The following version.lock entries have previous locked versions which are lower than the '
f'currently supported min_stack ({min_version}). To address this, run the '
f'`dev trim-version-lock {min_version}` command.\n\n{err_str}')