From c16442517ea2ed05064ccfcab23c0f325aaa954e Mon Sep 17 00:00:00 2001 From: Justin Ibarra Date: Thu, 2 Jun 2022 16:34:54 -0800 Subject: [PATCH] [Bug] Fix test_matrix_to_lock_version_defaults test (#2014) (cherry picked from commit e850f3952621e404c38846381b06db7b1349d844) --- detection_rules/devtools.py | 11 ++++++++--- detection_rules/schemas/__init__.py | 19 ++++++++++++++++--- detection_rules/version_lock.py | 5 ++--- tests/test_gh_workflows.py | 2 +- tests/test_version_locking.py | 4 ++-- 5 files changed, 29 insertions(+), 12 deletions(-) diff --git a/detection_rules/devtools.py b/detection_rules/devtools.py index c08f8fb10..5bd3912f5 100644 --- a/detection_rules/devtools.py +++ b/detection_rules/devtools.py @@ -824,9 +824,12 @@ def update_navigator_gists(directory: Path, token: str, gist_id: str, print_mark @dev_group.command('trim-version-lock') @click.argument('min_version') @click.option('--dry-run', is_flag=True, help='Print the changes rather than saving the file') -def trim_version_lock(min_version: str, dry_run: bool) -> dict: +def trim_version_lock(min_version: str, dry_run: bool): """Trim all previous entries within the version lock file which are lower than the min_version.""" - min_version = min(Version(v) for v in get_stack_versions(drop_patch=False)) + stack_versions = get_stack_versions() + assert min_version in stack_versions, f'Unknown min_version ({min_version}), expected: {", ".join(stack_versions)}' + + min_version = Version(min_version) version_lock_dict = default_version_lock.version_lock.to_dict() removed = {} @@ -843,7 +846,9 @@ def trim_version_lock(min_version: str, dry_run: bool) -> dict: latest_version = max(outdated_vers) if dry_run: - removed[rule_id] = [str(v) for v in outdated_vers] + outdated_minus_current = [str(v) for v in outdated_vers if v != min_version] + if outdated_minus_current: + removed[rule_id] = outdated_minus_current for outdated in outdated_vers: popped = lock['previous'].pop(str(outdated)) if outdated == latest_version: diff --git a/detection_rules/schemas/__init__.py b/detection_rules/schemas/__init__.py index 366ba31cc..4fa0b5ac3 100644 --- a/detection_rules/schemas/__init__.py +++ b/detection_rules/schemas/__init__.py @@ -19,6 +19,7 @@ __all__ = ( "SCHEMA_DIR", "definitions", "downgrade", + "get_min_supported_stack_version", "get_stack_schemas", "get_stack_versions", "validate_rta_mapping", @@ -228,6 +229,11 @@ def downgrade(api_contents: dict, target_version: str, current_version: Optional return api_contents +@cached +def load_stack_schema_map() -> dict: + return load_etc_dump('stack-schema-map.yaml') + + @cached def get_stack_schemas(stack_version: Optional[str] = '0.0.0') -> OrderedDictType[str, dict]: """Return all ECS + beats to stack versions for every stack version >= specified stack version and <= package.""" @@ -239,7 +245,7 @@ def get_stack_schemas(stack_version: Optional[str] = '0.0.0') -> OrderedDictType if len(current_package) == 2: current_package = Version(current_package + (0,)) - stack_map = load_etc_dump('stack-schema-map.yaml') + stack_map = load_stack_schema_map() versions = {k: v for k, v in stack_map.items() if (mapped_version := Version(k)) >= stack_version and mapped_version <= current_package and v} @@ -250,9 +256,9 @@ def get_stack_schemas(stack_version: Optional[str] = '0.0.0') -> OrderedDictType return versions_reversed -def get_stack_versions(drop_patch=True, reverse=True) -> List[str]: +def get_stack_versions(drop_patch=False) -> List[str]: """Get a list of stack versions supported (for the matrix).""" - versions = reversed(get_stack_schemas()) if reverse else list(get_stack_schemas()) + versions = list(load_stack_schema_map()) if drop_patch: abridged_versions = [] for version in versions: @@ -261,3 +267,10 @@ def get_stack_versions(drop_patch=True, reverse=True) -> List[str]: return abridged_versions else: return versions + + +def get_min_supported_stack_version() -> Version: + """Get the minimum defined and supported stack version.""" + stack_map = load_stack_schema_map() + min_version = min(Version(v) for v in list(stack_map)) + return min_version diff --git a/detection_rules/version_lock.py b/detection_rules/version_lock.py index f6a83440c..58a690e07 100644 --- a/detection_rules/version_lock.py +++ b/detection_rules/version_lock.py @@ -12,7 +12,7 @@ import click from .mixins import LockDataclassMixin, MarshmallowDataclassMixin from .rule_loader import RuleCollection -from .schemas import definitions, get_stack_versions +from .schemas import definitions, get_min_supported_stack_version from .semver import Version from .utils import cached, get_etc_path @@ -82,8 +82,7 @@ class DeprecatedRulesFile(LockDataclassMixin): def _convert_lock_version(stack_version: Optional[str]) -> Version: """Convert an optional stack version to the minimum for the lock.""" - versions = get_stack_versions(drop_patch=False) - min_version = min(Version(v) for v in versions) + min_version = get_min_supported_stack_version() if stack_version is None: return min_version return max(Version(stack_version), min_version) diff --git a/tests/test_gh_workflows.py b/tests/test_gh_workflows.py index 076bcb9b2..0aee5322d 100644 --- a/tests/test_gh_workflows.py +++ b/tests/test_gh_workflows.py @@ -26,6 +26,6 @@ class TestWorkflows(unittest.TestCase): lock_workflow = yaml.safe_load(lock_workflow_file.read_text()) lock_versions = lock_workflow[True]['workflow_dispatch']['inputs']['branches']['default'].split(',') - matrix_versions = get_stack_versions() + matrix_versions = get_stack_versions(drop_patch=True) err_msg = 'lock-versions workflow default does not match current matrix in stack-schema-map' self.assertListEqual(lock_versions, matrix_versions[:-1], err_msg) diff --git a/tests/test_version_locking.py b/tests/test_version_locking.py index 148762b9b..9547b1825 100644 --- a/tests/test_version_locking.py +++ b/tests/test_version_locking.py @@ -7,7 +7,7 @@ import unittest -from detection_rules.schemas import get_stack_versions +from detection_rules.schemas import get_min_supported_stack_version from detection_rules.semver import Version from detection_rules.version_lock import default_version_lock @@ -18,7 +18,7 @@ class TestVersionLock(unittest.TestCase): def test_previous_entries_gte_current_min_stack(self): """Test that all previous entries for all locks in the version lock are >= the current min_stack.""" errors = {} - min_version = min(Version(v) for v in get_stack_versions(drop_patch=False)) + min_version = get_min_supported_stack_version() for rule_id, lock in default_version_lock.version_lock.to_dict().items(): if 'previous' in lock: prev_vers = [Version(v) for v in list(lock['previous'])]