Bumping number of versions per rule to 4 in total (#4451)

* Bumping number of versions per rule to 4 in total

* Add explicit caps

* Simpler comment

* Renaming constants

* Drop to 8.17 again

* Clearer constants

* Drop if condition and extend the comment

* Shorten the lines

* Version bump

---------

Co-authored-by: Mika Ayenson <Mikaayenson@users.noreply.github.com>

detection_rules/devtools.py

@@ -63,8 +63,13 @@ NAVIGATOR_BADGE = (
     f'[![ATT&CK navigator coverage](https://img.shields.io/badge/ATT&CK-Navigator-red.svg)]({NAVIGATOR_URL})'
 )
 RULES_CONFIG = parse_rules_config()
-# The base package version that we will start to include all versions of historical rules
-BASE_PKG_VERSION = Version(major=8, minor=17, patch=0)
+
+# The rule diff feature is available in 8.18 but needs to be tested in pre-release versions
+MIN_DIFF_FEATURE_VERSION = Version(major=8, minor=17, patch=0)
+
+# The caps for the historical versions of the rules
+MAX_HISTORICAL_VERSIONS_FOR_DIFF = 3
+MAX_HISTORICAL_VERSIONS_PRE_DIFF = 1
 
 
 def get_github_token() -> Optional[str]:
@@ -131,14 +136,17 @@ def build_release(ctx: click.Context, config_file, update_version_lock: bool, ge
     # Version 8.17.0-beta.1 is considered lower than 8.17.0
     current_pkg_version_no_prerelease = Version(major=current_pkg_version.major,
                                                 minor=current_pkg_version.minor, patch=current_pkg_version.patch)
-    if current_pkg_version_no_prerelease >= BASE_PKG_VERSION:
-        click.echo(f'[+] Adding all historical rule versions in our release package for version \
-            {current_pkg_version_no_prerelease}')
-        limited_historical_rules = historical_rules
-    else:
-        click.echo(f'[+] Limit historical rule versions in our release package for version \
-            {current_pkg_version_no_prerelease}')
-        limited_historical_rules = sde.keep_latest_versions(historical_rules)
+
+    hist_versions_num = (
+        MAX_HISTORICAL_VERSIONS_FOR_DIFF
+        if current_pkg_version_no_prerelease >= MIN_DIFF_FEATURE_VERSION
+        else MAX_HISTORICAL_VERSIONS_PRE_DIFF
+    )
+    click.echo(
+        '[+] Limit historical rule versions in the release package for '
+        f'version {current_pkg_version_no_prerelease}: {hist_versions_num} versions')
+    limited_historical_rules = sde.keep_latest_versions(historical_rules, num_versions=hist_versions_num)
+
     package.add_historical_rules(limited_historical_rules, registry_data['version'])
     click.echo(f'[+] Adding historical rules from {previous_pkg_version} package')
 

detection_rules/integrations.py

@@ -26,7 +26,7 @@ from .utils import cached, get_etc_path, read_gzip, unzip
 from .schemas import definitions
 
 MANIFEST_FILE_PATH = get_etc_path('integration-manifests.json.gz')
-NUM_LATEST_RULE_VERSIONS = 1
+DEFAULT_MAX_RULE_VERSIONS = 1
 SCHEMA_FILE_PATH = get_etc_path('integration-schemas.json.gz')
 _notified_integrations = set()
 
@@ -417,7 +417,7 @@ class SecurityDetectionEngine:
                       for x in asset_file_names}
         return assets
 
-    def keep_latest_versions(self, assets: dict, num_versions: int = NUM_LATEST_RULE_VERSIONS) -> dict:
+    def keep_latest_versions(self, assets: dict, num_versions: int = DEFAULT_MAX_RULE_VERSIONS) -> dict:
         """Keeps only the latest N versions of each rule to limit historical rule versions in our release package."""
 
         # Dictionary to hold the sorted list of versions for each base rule ID

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "detection_rules"
-version = "0.4.8"
+version = "0.4.9"
 description = "Detection Rules is the home for rules used by Elastic Security. This repository is used for the development, maintenance, testing, validation, and release of rules for Elastic Security’s Detection Engine."
 readme = "README.md"
 requires-python = ">=3.12"