1554 update eql schemas to fail validation on text fields (#1866)

* Ensure kql2eql conversion doesnt support `text` fields

* Add unit test cases for`text` not supported in eql

* test `field not recognized` in the rule_validator and output a verbose message.

* use elasticsearch_type_family to lookup text mappings

Co-authored-by: Justin Ibarra <brokensound77@users.noreply.github.com>

(cherry picked from commit 1f015ebe85)

detection_rules/rule_validators.py

@@ -67,6 +67,12 @@ class EQLValidator(QueryValidator):
         with eql.parser.elasticsearch_syntax, eql.parser.ignore_missing_functions:
             return eql.parse_query(self.query)
 
+    def text_fields(self, eql_schema: ecs.KqlSchema2Eql) -> List[str]:
+        """Return a list of fields of type text."""
+        from kql.parser import elasticsearch_type_family
+
+        return [f for f in self.unique_fields if elasticsearch_type_family(eql_schema.kql_schema.get(f)) == 'text']
+
     @property
     def unique_fields(self) -> List[str]:
         return list(set(str(f) for f in self.ast if isinstance(f, eql.ast.Field)))
@@ -98,6 +104,11 @@ class EQLValidator(QueryValidator):
                 trailer = err_trailer
                 if "Unknown field" in message and beat_types:
                     trailer = f"\nTry adding event.module or event.dataset to specify beats module\n\n{trailer}"
+                elif "Field not recognized" in message:
+                    text_fields = self.text_fields(eql_schema)
+                    if text_fields:
+                        fields_str = ', '.join(text_fields)
+                        trailer = f"\neql does not support text fields: {fields_str}\n\n{trailer}"
 
                 raise exc.__class__(exc.error_msg, exc.line, exc.column, exc.source,
                                     len(exc.caret.lstrip()), trailer=trailer) from None

kql/kql2eql.py

@@ -7,6 +7,8 @@ import eql
 
 from .parser import BaseKqlParser
 
+NOT_SUPPORTED_EQL_FIELDS = ["text"]
+#  https://github.com/elastic/eql/issues/17
 
 class KqlToEQL(BaseKqlParser):
 
@@ -51,7 +53,12 @@ class KqlToEQL(BaseKqlParser):
 
         with self.scope(self.visit(field_tree)) as field_name:
             # check the field against the schema
-            self.get_field_type(field_name, field_tree)
+
+            type_mapping = self.get_field_type(field_name, field_tree)
+            if type_mapping in NOT_SUPPORTED_EQL_FIELDS:
+                err_msg = f"{field_name} uses an unsupported elasticsearch eql field_type {type_mapping}"
+                raise eql.EqlSemanticError(err_msg, field_tree.line, field_tree.column, self.text)
+
             return self.visit(value_tree)
 
     def or_list_of_values(self, tree):

tests/kuery/test_kql2eql.py

@@ -73,13 +73,17 @@ class TestKql2Eql(unittest.TestCase):
         self.validate("top.numF : 1", "top.numF == 1", schema=schema)
         self.validate("top.numF : \"1\"", "top.numF == 1", schema=schema)
         self.validate("top.keyword : 1", "top.keyword == '1'", schema=schema)
-        self.validate("top.text : \"hello\"", "top.text == 'hello'", schema=schema)
         self.validate("top.keyword : \"hello\"", "top.keyword == 'hello'", schema=schema)
-        self.validate("top.text : 1 ", "top.text == '1'", schema=schema)
         self.validate("dest:192.168.255.255", "dest == '192.168.255.255'", schema=schema)
         self.validate("dest:192.168.0.0/16", "cidrMatch(dest, '192.168.0.0/16')", schema=schema)
         self.validate("dest:\"192.168.0.0/16\"", "cidrMatch(dest, '192.168.0.0/16')", schema=schema)
 
+        with self.assertRaises(eql.EqlSemanticError):
+            self.validate("top.text : \"hello\"", "top.text == 'hello'", schema=schema)
+
+        with self.assertRaises(eql.EqlSemanticError):
+            self.validate("top.text : 1 ", "top.text == '1'", schema=schema)
+
         with self.assertRaisesRegex(kql.KqlParseError, r"Value doesn't match top.middle's type: nested"):
             kql.to_eql("top.middle : 1", schema=schema)
 

tests/test_schemas.py

@@ -208,6 +208,18 @@ class TestSchemas(unittest.TestCase):
             process where process.name == "cmd.exe"
         """)
 
+        example_text_fields = ['client.as.organization.name.text', 'client.user.full_name.text',
+                               'client.user.name.text', 'destination.as.organization.name.text',
+                               'destination.user.full_name.text', 'destination.user.name.text',
+                               'error.message', 'error.stack_trace.text', 'file.path.text',
+                               'file.target_path.text', 'host.os.full.text', 'host.os.name.text',
+                               'host.user.full_name.text', 'host.user.name.text']
+        for text_field in example_text_fields:
+            with self.assertRaises(eql.parser.EqlSchemaError):
+                build_rule(f"""
+                        any where {text_field} == "some string field"
+                """)
+
         with self.assertRaises(eql.EqlSyntaxError):
             build_rule("""
                     process where process.name == this!is$not#v@lid