[Bug] Omit ES|QL engine columns from required_fields (#6027)

* Omit Esql.* columns from ES|QL rule required_fields

Kibana treats required_fields as index mappings. ES|QL stats and
similar commands expose Esql.* and Esql_priv.* result columns that
are not mapped on source indices, which produced noisy validation
warnings for shipped rules.

Filter those names when building required_fields. Add a check in
test_esql_endpoint_alerts_index when remote ES|QL validation runs.

Fixes #6026.

* Move required_fields check to its own remote test

* Iterate production rules in required_fields test

* Use direct get_required_fields call in remote test

Skip to_api_format() and call data.get_required_fields(index) directly,
gated on ESQLRuleData. Mirrors the ESQLValidator scope of the fix and
avoids the unrelated packaging steps that to_api_format runs per rule.

* Bump version to 1.6.30

* Centralize ES|QL dynamic field prefix tuple

Define ESQL_DYNAMIC_FIELD_PREFIXES = ("Esql.", "Esql_priv.") in
schemas/definitions.py and reuse it in QueryValidator.get_required_fields,
ESQLValidator.validate_columns_index_mapping, and the remote test.
Single source of truth and consistent ordering across the codebase.

detection_rules/rule.py

@@ -682,6 +682,8 @@ class QueryValidator:
 
         required: list[dict[str, Any]] = []
         unique_fields: list[str] = self.unique_fields or []
+        if isinstance(self, ESQLValidator):
+            unique_fields = [f for f in unique_fields if not f.startswith(definitions.ESQL_DYNAMIC_FIELD_PREFIXES)]
 
         for fld in unique_fields:
             field_type = ecs_schema.get(fld, {}).get("type")

detection_rules/rule_validators.py

@@ -47,7 +47,7 @@ from .integrations import (
 )
 from .rule import EQLRuleData, QueryRuleData, QueryValidator, RuleMeta, TOMLRuleContents, set_eql_config
 from .schemas import get_latest_stack_version, get_stack_schemas, get_stack_versions
-from .schemas.definitions import FROM_SOURCES_REGEX
+from .schemas.definitions import ESQL_DYNAMIC_FIELD_PREFIXES, FROM_SOURCES_REGEX
 
 EQL_ERROR_TYPES = (
     eql.EqlCompileError
@@ -792,7 +792,7 @@ class ESQLValidator(QueryValidator):
         for column in query_columns:
             column_name = column["name"]
             # Skip Dynamic fields
-            if column_name.startswith(("Esql.", "Esql_priv.")):
+            if column_name.startswith(ESQL_DYNAMIC_FIELD_PREFIXES):
                 continue
             # Skip internal fields
             if column_name in ("_id", "_version", "_index"):

detection_rules/schemas/definitions.py

@@ -79,6 +79,7 @@ MINOR_SEMVER = re.compile(r"^\d+\.\d+$")
 FROM_SOURCES_REGEX = re.compile(
     r"^\s*FROM\s+(?P<sources>(?:.+?(?:,\s*)?\n?)+?)\s*(?:\||\bmetadata\b|//|$)", re.IGNORECASE | re.MULTILINE
 )
+ESQL_DYNAMIC_FIELD_PREFIXES = ("Esql.", "Esql_priv.")
 BRANCH_PATTERN = f"{VERSION_PATTERN}|^master$"
 ELASTICSEARCH_EQL_FEATURES = {
     "allow_negation": (Version.parse("8.9.0"), None),

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "detection_rules"
-version = "1.6.29"
+version = "1.6.30"
 description = "Detection Rules is the home for rules used by Elastic Security. This repository is used for the development, maintenance, testing, validation, and release of rules for Elastic Security’s Detection Engine."
 readme = "README.md"
 requires-python = ">=3.12"

tests/test_rules_remote.py

@@ -19,7 +19,9 @@ from detection_rules.misc import (
     get_default_config,
     getdefault,
 )
+from detection_rules.rule import ESQLRuleData
 from detection_rules.rule_loader import RuleCollection
+from detection_rules.schemas.definitions import ESQL_DYNAMIC_FIELD_PREFIXES
 from detection_rules.utils import get_path, load_rule_contents
 
 from .base import BaseRuleTest
@@ -244,6 +246,20 @@ class TestRemoteRules(BaseRuleTest):
         """
         _ = RuleCollection().load_dict(production_rule)
 
+    def test_esql_required_fields_omit_engine_columns(self):
+        """ESQL required_fields must not list Esql.* / Esql_priv.* (not index mappings)."""
+        for rule in self.all_rules:
+            data = rule.contents.data
+            if not isinstance(data, ESQLRuleData):
+                continue
+            index = data.get("index") or []
+            for rf in data.get_required_fields(index) or []:
+                name = rf["name"]
+                assert not name.startswith(ESQL_DYNAMIC_FIELD_PREFIXES), (
+                    f"{rule.id} - {rule.name}: required_fields must not include ES|QL engine columns "
+                    f"(not index mappings): {name!r}"
+                )
+
     def test_esql_endpoint_unknown_index(self):
         """Test an ESQL rule's index validation. This is expected to error on an unknown index."""
         file_path = get_path(["tests", "data", "command_control_dummy_production_rule.toml"])