fix: Cleaning up the hashable content for the rule (#4621)

detection_rules/packaging.py

@@ -465,7 +465,7 @@ class Package(object):
         for rule in self.rules:
             summary_doc['rule_ids'].append(rule.id)
             summary_doc['rule_names'].append(rule.name)
-            summary_doc['rule_hashes'].append(rule.contents.sha256())
+            summary_doc['rule_hashes'].append(rule.contents.get_hash())
 
             if rule.id in self.new_ids:
                 status = 'new'
@@ -481,7 +481,7 @@ class Package(object):
             if relative_path is None:
                 raise ValueError(f"Could not find a valid relative path for the rule: {rule.id}")
 
-            rule_doc = dict(hash=rule.contents.sha256(),
+            rule_doc = dict(hash=rule.contents.get_hash(),
                             source='repo',
                             datetime_uploaded=now,
                             status=status,

detection_rules/rule.py

@@ -1010,18 +1010,25 @@ class BaseRuleContents(ABC):
 
     def lock_info(self, bump=True) -> dict:
         version = self.autobumped_version if bump else (self.saved_version or 1)
-        contents = {"rule_name": self.name, "sha256": self.sha256(), "version": version, "type": self.type}
+        contents = {"rule_name": self.name, "sha256": self.get_hash(), "version": version, "type": self.type}
 
         return contents
 
     @property
-    def is_dirty(self) -> Optional[bool]:
+    def is_dirty(self) -> bool:
         """Determine if the rule has changed since its version was locked."""
         min_stack = Version.parse(self.get_supported_version(), optional_minor_and_patch=True)
         existing_sha256 = self.version_lock.get_locked_hash(self.id, f"{min_stack.major}.{min_stack.minor}")
 
-        if existing_sha256 is not None:
-            return existing_sha256 != self.sha256()
+        if not existing_sha256:
+            return False
+
+        rule_hash = self.get_hash()
+        rule_hash_with_integrations = self.get_hash(include_integrations=True)
+
+        # Checking against current and previous version of the hash to avoid mass version bump
+        is_dirty = existing_sha256 not in (rule_hash, rule_hash_with_integrations)
+        return is_dirty
 
     @property
     def lock_entry(self) -> Optional[dict]:
@@ -1123,10 +1130,25 @@ class BaseRuleContents(ABC):
     def to_api_format(self, include_version: bool = True) -> dict:
         """Convert the rule to the API format."""
 
+    def get_hashable_content(self, include_version: bool = False, include_integrations: bool = False) -> dict:
+        """Returns the rule content to be used for calculating the hash value for the rule"""
+
+        # get the API dict without the version by default, otherwise it'll always be dirty.
+        hashable_dict = self.to_api_format(include_version=include_version)
+
+        # drop related integrations if present
+        if not include_integrations:
+            hashable_dict.pop("related_integrations", None)
+
+        return hashable_dict
+
     @cached
-    def sha256(self, include_version=False) -> str:
-        # get the hash of the API dict without the version by default, otherwise it'll always be dirty.
-        hashable_contents = self.to_api_format(include_version=include_version)
+    def get_hash(self, include_version: bool = False, include_integrations: bool = False) -> str:
+        """Returns a sha256 hash of the rule contents"""
+        hashable_contents = self.get_hashable_content(
+            include_version=include_version,
+            include_integrations=include_integrations,
+        )
         return utils.dict_hash(hashable_contents)
 
 

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "detection_rules"
-version = "1.0.18"
+version = "1.1.0"
 description = "Detection Rules is the home for rules used by Elastic Security. This repository is used for the development, maintenance, testing, validation, and release of rules for Elastic Security’s Detection Engine."
 readme = "README.md"
 requires-python = ">=3.12"

tests/test_packages.py

@@ -43,7 +43,7 @@ class TestPackages(BaseRuleTest):
         version_info = {
             rule.id: {
                 'rule_name': rule.name,
-                'sha256': rule.contents.sha256(),
+                'sha256': rule.contents.get_hash(),
                 'version': version
             } for rule in rules
         }
@@ -76,7 +76,7 @@ class TestPackages(BaseRuleTest):
         # test that no rules have versions defined
         for rule in rules:
             self.assertGreaterEqual(rule.contents.autobumped_version, 1, '{} - {}: version is not being set in package')
-            original_hashes.append(rule.contents.sha256())
+            original_hashes.append(rule.contents.get_hash())
 
         package = Package(rules, 'test-package')
 
@@ -87,7 +87,7 @@ class TestPackages(BaseRuleTest):
 
         # test that rules validate with version
         for rule in package.rules:
-            post_bump_hashes.append(rule.contents.sha256())
+            post_bump_hashes.append(rule.contents.get_hash())
 
         # test that no hashes changed as a result of the version bumps
         self.assertListEqual(original_hashes, post_bump_hashes, 'Version bumping modified the hash of a rule')