diff --git a/detection_rules/rule_validators.py b/detection_rules/rule_validators.py index 49031d2ad..aaed3de5e 100644 --- a/detection_rules/rule_validators.py +++ b/detection_rules/rule_validators.py @@ -67,6 +67,12 @@ class EQLValidator(QueryValidator): with eql.parser.elasticsearch_syntax, eql.parser.ignore_missing_functions: return eql.parse_query(self.query) + def text_fields(self, eql_schema: ecs.KqlSchema2Eql) -> List[str]: + """Return a list of fields of type text.""" + from kql.parser import elasticsearch_type_family + + return [f for f in self.unique_fields if elasticsearch_type_family(eql_schema.kql_schema.get(f)) == 'text'] + @property def unique_fields(self) -> List[str]: return list(set(str(f) for f in self.ast if isinstance(f, eql.ast.Field))) @@ -98,6 +104,11 @@ class EQLValidator(QueryValidator): trailer = err_trailer if "Unknown field" in message and beat_types: trailer = f"\nTry adding event.module or event.dataset to specify beats module\n\n{trailer}" + elif "Field not recognized" in message: + text_fields = self.text_fields(eql_schema) + if text_fields: + fields_str = ', '.join(text_fields) + trailer = f"\neql does not support text fields: {fields_str}\n\n{trailer}" raise exc.__class__(exc.error_msg, exc.line, exc.column, exc.source, len(exc.caret.lstrip()), trailer=trailer) from None diff --git a/kql/kql2eql.py b/kql/kql2eql.py index 07d95a089..7a275c00f 100755 --- a/kql/kql2eql.py +++ b/kql/kql2eql.py @@ -7,6 +7,8 @@ import eql from .parser import BaseKqlParser +NOT_SUPPORTED_EQL_FIELDS = ["text"] +# https://github.com/elastic/eql/issues/17 class KqlToEQL(BaseKqlParser): @@ -51,7 +53,12 @@ class KqlToEQL(BaseKqlParser): with self.scope(self.visit(field_tree)) as field_name: # check the field against the schema - self.get_field_type(field_name, field_tree) + + type_mapping = self.get_field_type(field_name, field_tree) + if type_mapping in NOT_SUPPORTED_EQL_FIELDS: + err_msg = f"{field_name} uses an unsupported elasticsearch eql field_type {type_mapping}" + raise eql.EqlSemanticError(err_msg, field_tree.line, field_tree.column, self.text) + return self.visit(value_tree) def or_list_of_values(self, tree): diff --git a/tests/kuery/test_kql2eql.py b/tests/kuery/test_kql2eql.py index 6aaccb8e6..bfa9a2425 100644 --- a/tests/kuery/test_kql2eql.py +++ b/tests/kuery/test_kql2eql.py @@ -73,13 +73,17 @@ class TestKql2Eql(unittest.TestCase): self.validate("top.numF : 1", "top.numF == 1", schema=schema) self.validate("top.numF : \"1\"", "top.numF == 1", schema=schema) self.validate("top.keyword : 1", "top.keyword == '1'", schema=schema) - self.validate("top.text : \"hello\"", "top.text == 'hello'", schema=schema) self.validate("top.keyword : \"hello\"", "top.keyword == 'hello'", schema=schema) - self.validate("top.text : 1 ", "top.text == '1'", schema=schema) self.validate("dest:192.168.255.255", "dest == '192.168.255.255'", schema=schema) self.validate("dest:192.168.0.0/16", "cidrMatch(dest, '192.168.0.0/16')", schema=schema) self.validate("dest:\"192.168.0.0/16\"", "cidrMatch(dest, '192.168.0.0/16')", schema=schema) + with self.assertRaises(eql.EqlSemanticError): + self.validate("top.text : \"hello\"", "top.text == 'hello'", schema=schema) + + with self.assertRaises(eql.EqlSemanticError): + self.validate("top.text : 1 ", "top.text == '1'", schema=schema) + with self.assertRaisesRegex(kql.KqlParseError, r"Value doesn't match top.middle's type: nested"): kql.to_eql("top.middle : 1", schema=schema) diff --git a/tests/test_schemas.py b/tests/test_schemas.py index 499cf53bc..dce9042da 100644 --- a/tests/test_schemas.py +++ b/tests/test_schemas.py @@ -208,6 +208,18 @@ class TestSchemas(unittest.TestCase): process where process.name == "cmd.exe" """) + example_text_fields = ['client.as.organization.name.text', 'client.user.full_name.text', + 'client.user.name.text', 'destination.as.organization.name.text', + 'destination.user.full_name.text', 'destination.user.name.text', + 'error.message', 'error.stack_trace.text', 'file.path.text', + 'file.target_path.text', 'host.os.full.text', 'host.os.name.text', + 'host.user.full_name.text', 'host.user.name.text'] + for text_field in example_text_fields: + with self.assertRaises(eql.parser.EqlSchemaError): + build_rule(f""" + any where {text_field} == "some string field" + """) + with self.assertRaises(eql.EqlSyntaxError): build_rule(""" process where process.name == this!is$not#v@lid