[FR] Add ESQL rules to dataset exception (#5249)

* Add ESQL rules to dataset exception

* Add unit test

detection_rules/index_mappings.py

@@ -261,7 +261,6 @@ def get_filtered_index_schema(
     filtered_keys.update(non_ecs_indices.keys())
     filtered_keys.update(custom_indices.keys())
     filtered_keys.add("logs-endpoint.alerts-*")
-    filtered_keys.update(indices)
 
     matches: list[str] = []
     for index in indices:

detection_rules/rule.py

@@ -1528,7 +1528,11 @@ class TOMLRuleContents(BaseRuleContents, MarshmallowDataclassMixin):
                 *definitions.NON_DATASET_PACKAGES,
                 *map(str.lower, definitions.MACHINE_LEARNING_PACKAGES),
             ]
-            if integration in ineligible_integrations or isinstance(data, MachineLearningRuleData):
+            if (
+                integration in ineligible_integrations
+                or isinstance(data, MachineLearningRuleData)
+                or (isinstance(data, ESQLRuleData) and integration not in datasets)
+            ):
                 packaged_integrations.append({"package": integration, "integration": None})
 
         packaged_integrations.extend(parse_datasets(list(datasets), package_manifest))

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "detection_rules"
-version = "1.5.4"
+version = "1.5.5"
 description = "Detection Rules is the home for rules used by Elastic Security. This repository is used for the development, maintenance, testing, validation, and release of rules for Elastic Security’s Detection Engine."
 readme = "README.md"
 requires-python = ">=3.12"

tests/test_rules_remote.py

@@ -46,6 +46,28 @@ class TestRemoteRules(BaseRuleTest):
         for integration in related_integrations:
             assert integration["package"] == "aws", f"Expected 'aws', but got {integration['package']}"
 
+    def test_esql_non_dataset_package_related_integrations(self):
+        """Test an ESQL rule has its related integrations built correctly with a non dataset package."""
+        file_path = get_path(["tests", "data", "command_control_dummy_production_rule.toml"])
+        original_production_rule = load_rule_contents(file_path)
+        production_rule = deepcopy(original_production_rule)[0]
+        production_rule["metadata"]["integration"] = ["aws_bedrock"]
+        production_rule["rule"]["query"] = """
+        from logs-aws_bedrock.invocation-* metadata _id, _version, _index
+        // Filter for access denied errors from GenAI responses
+        | where gen_ai.response.error_code == "AccessDeniedException"
+        // keep ECS and response fields
+        | keep
+        user.id,
+        gen_ai.request.model.id,
+        cloud.account.id,
+        gen_ai.response.error_code
+        """
+        rule = RuleCollection().load_dict(production_rule)
+        related_integrations = rule.contents.to_api_format()["related_integrations"]
+        for integration in related_integrations:
+            assert integration["package"] == "aws_bedrock", f"Expected 'aws_bedrock', but got {integration['package']}"
+
     def test_esql_event_dataset_schema_error(self):
         """Test an ESQL rule that uses event.dataset field in the query that restricts the schema failing validation."""
         file_path = get_path(["tests", "data", "command_control_dummy_production_rule.toml"])