From d3ba1e4fb8e97981279de8d295645c6c38893666 Mon Sep 17 00:00:00 2001 From: SOC Prime <37212749+socprime@users.noreply.github.com> Date: Tue, 18 Aug 2020 11:20:22 +0300 Subject: [PATCH 01/11] Add sysmon backend --- tools/sigma/backends/sysmon.py | 250 +++++++++++++++++++++++++++++++++ 1 file changed, 250 insertions(+) create mode 100644 tools/sigma/backends/sysmon.py diff --git a/tools/sigma/backends/sysmon.py b/tools/sigma/backends/sysmon.py new file mode 100644 index 000000000..3c8c74389 --- /dev/null +++ b/tools/sigma/backends/sysmon.py @@ -0,0 +1,250 @@ +# Output backends for sigmac +# Copyright 2020 SOC Prime + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. + +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + + +import re + +import sigma +from sigma.backends.base import SingleTextQueryBackend +from sigma.backends.mixins import MultiRuleOutputMixin + + +class SysmonConfigBackend(SingleTextQueryBackend, MultiRuleOutputMixin): + identifier = "sysmon" + active = True + andToken = " AND " + orToken = " OR " + notToken = "NOT " + subExpression = "(%s)" + config_required = False + INCLUDE = "include" + EXCLUDE = "exclude" + conditionDict = { + "startswith": "begin with", + "endswith": "end with", + } + + def __init__(self, *args, **kwargs): + self.table = None + self.logsource = None + self.allowedSource = { + "process_creation": "ProcessCreate" + } + self.eventidTagMapping = { + 1: "ProcessCreate", + 2: "FileCreateTime", + 3: "NetworkConnect", + 5: "ProcessTerminate", + 6: "DriverLoad", + 7: "ImageLoad", + 8: "CreateRemoteThread", + 9: "RawAccessRead", + 10: "ProcessAccess", + 11: "FileCreate", + 12: "RegistryEvent", + 13: "RegistryEvent", + 14: "RegistryEvent", + 15: "FileCreateStreamHash", + 17: "PipeEvent", + 18: "PipeEvent", + 19: "WmiEvent", + 20: "WmiEvent", + 21: "WmiEvent", + 22: "DNSQuery", + 23: "FileDelete" + } + self.allowedCondCombinations = { + 'single': [ + [4], + [1, 4], + [2, 4], + ], + 'multi': [ + [1, 2, 4], + ], + "exclude": [ + [1, 3, 4], + [2, 3, 4] + ], + # "multi-exclude": [ + # [1, 2, 3, 4] + # ] + } + return super().__init__(*args, **kwargs) + + def cleanValue(self, value): + val = re.sub("[*]", "", value) + return val + + def mapFiledValue(self, field, value): + condition = None + if "|" in field: + field, *pipes = field.split("|") + if len(pipes) == 1: + condition = pipes[0] + else: + raise NotImplementedError("not implemented condition") + if isinstance(value, list) and len(value) > 1: + condition = "contains any" + value = ";".join(value) + elif "*" in value: + if value.startswith("*") and value.endswith("*"): + condition = "contains" + elif value.startswith("*"): + condition = "end with" + elif value.endswith("*"): + condition = "begin with" + else: + condition = "contains" + + if condition: + field_str = '<{field} condition="{condition}">{value}'.format(field=field, + condition=condition, + value=self.cleanValue(value)) + else: + field_str = '<{field}>{value}'.format(field=field, value=self.cleanValue(value)) + + return field_str + + def createRule(self, selections): + fields_list = [] + table = None + for field, value in selections.items(): + if isinstance(value, list) and len(value) == 1: + value = value[0] + if field == "EventID": + table = self.eventidTagMapping[value] + else: + created_field_value = self.mapFiledValue(field, value) + fields_list.append(created_field_value) + fields_list_filtered = [item for item in fields_list if item] + if any(fields_list_filtered): + rule = '''\n\t\t\n\t\t\t{fields}\n\t\t'''.format(rule_name=self.rule_name, fields="\n\t\t\t".join(["{}".format(item) for item in fields_list_filtered])) + t = table if table else self.table + return rule, t + else: + return None, None + + def createRuleGroup(self, condition_objects, condition, match_type="include"): + rules = None + rules_selections = [item for item in condition_objects if item.type == 4] + if len(rules_selections) == 1: + rule, table = self.createRule(self.detection.get(rules_selections[0].matched)) + rules = {match_type: {table: rule}} + else: + if "or" in condition.lower(): + result = {} + for selection_object in rules_selections: + rule, table = self.createRule(self.detection.get(selection_object.matched)) + if result.get(table): + result[table].append(rule) + else: + result[table] = [rule] + result = {table_name: "\n\t\t".join(rules_list) for table_name, rules_list in result.items()} + rules = {match_type: result} + elif "and" in condition.lower(): + rules_dict = {} + for selection_object in rules_selections: + rules_dict.update(self.detection.get(selection_object.matched)) + rule, table = self.createRule(rules_dict) + rules = {match_type: {table: rule}} + if rules: + rules_result = [] + for match, tables in rules.items(): + for table, rules in tables.items(): + category_comment = '\n\n{}'.format(table, match, + "".join(rules)) + rules_result.append(category_comment) + return "".join(rules_result) + else: + raise + + def createMultiRuleGroup(self, conditions): + conditions_id = "".join([str(item.type) for item in conditions]) + or_index = conditions_id.index("2") + sorted_conditions = [conditions[:or_index], conditions[or_index+1:]] + if sorted_conditions: + result = "" + for rule_condition in sorted_conditions: + rule = self.createRuleGroup(condition_objects=rule_condition, condition=" ".join([item.matched for item in rule_condition])) + result += "{}\n".format(rule) + return result + else: + raise NotImplementedError("not implemented condition") + + def createExcludeRuleGroup(self, conditions): + conditions_id = "".join([str(item.type) for item in conditions]) + condition = self.detection.get("condition") + sorted_conditions = None + if "and not" in condition.lower(): + andnot_index = conditions_id.index("13") + sorted_conditions = [(conditions[:andnot_index], self.INCLUDE), ([item for item in conditions if item.type != 3], self.EXCLUDE)] + elif "or not" in condition.lower(): + ornot_index = conditions_id.index("23") + sorted_conditions = [(conditions[:ornot_index], self.INCLUDE), (conditions[ornot_index + 2:], self.EXCLUDE)] + if sorted_conditions: + result = "" + for rule_condition in sorted_conditions: + rule = self.createRuleGroup(condition_objects=rule_condition[0], condition=" ".join([item.matched for item in rule_condition[0]]), match_type=rule_condition[1]) + result += "{}\n".format(rule) + return result + + def createMultiExcludeRuleGroup(self, conditions): + return + + def checkRuleCondition(self, condtokens): + if len(condtokens) == 1: + conditions = [item for item in condtokens[0].tokens] + conditions_combination = list(set([item.type for item in conditions])) + for rule_type, combinations in self.allowedCondCombinations.items(): + for combination in combinations: + if sorted(conditions_combination) == sorted(combination): + return rule_type, conditions + else: + raise NotImplementedError("not implemented condition") + else: + raise NotImplementedError("not implemented condition") + + def createTableFromLogsource(self): + if self.logsource.get("product", "") != "windows": + raise TypeError( + "Not supported logsource. Should be product windows") + for item in self.logsource.values(): + if item.lower() in self.allowedSource.keys(): + self.table = self.allowedSource.get(item.lower()) + break + else: + self.table = "ProcessCreate" + + def finalize(self): + rulegroup_comment = '' + return "{}\n{}".format(rulegroup_comment, self.sysmon_rule) + + + def generate(self, sigmaparser): + title = sigmaparser.parsedyaml.get("title", "") + author = sigmaparser.parsedyaml.get("author", {}) + self.rule_name = "{} by {}".format(title, author) + self.detection = sigmaparser.parsedyaml.get("detection", {}) + self.logsource = sigmaparser.parsedyaml["logsource"] + self.createTableFromLogsource() + rule_type, conditions = self.checkRuleCondition(sigmaparser.condtoken) + if rule_type == "single": + self.sysmon_rule = self.createRuleGroup(conditions, self.detection.get("condition")) + elif rule_type == "multi": + self.sysmon_rule = self.createMultiRuleGroup(conditions) + elif rule_type == "exclude": + self.sysmon_rule = self.createExcludeRuleGroup(conditions) From a2fec9f3b9f8153f60b3f56b40a3a6fc9b1b666f Mon Sep 17 00:00:00 2001 From: vh Date: Fri, 28 Aug 2020 12:26:40 +0300 Subject: [PATCH 02/11] Fix sysmon backend --- ...g_sensitive_files_with_credential_data.yml | 42 -- .../win_susp_process_creations.yml | 74 --- tools/sigma/backends/sysmon.py | 494 +++++++++--------- 3 files changed, 244 insertions(+), 366 deletions(-) delete mode 100644 rules/windows/process_creation/win_copying_sensitive_files_with_credential_data.yml delete mode 100644 rules/windows/process_creation/win_susp_process_creations.yml diff --git a/rules/windows/process_creation/win_copying_sensitive_files_with_credential_data.yml b/rules/windows/process_creation/win_copying_sensitive_files_with_credential_data.yml deleted file mode 100644 index eb7818e2f..000000000 --- a/rules/windows/process_creation/win_copying_sensitive_files_with_credential_data.yml +++ /dev/null @@ -1,42 +0,0 @@ -title: Copying Sensitive Files with Credential Data -id: e7be6119-fc37-43f0-ad4f-1f3f99be2f9f -description: Files with well-known filenames (sensitive files with credential data) copying -status: experimental -author: Teymur Kheirkhabarov, Daniil Yugoslavskiy, oscd.community -date: 2019/10/22 -modified: 2019/11/13 -references: - - https://room362.com/post/2013/2013-06-10-volume-shadow-copy-ntdsdit-domain-hashes-remotely-part-1/ - - https://www.slideshare.net/heirhabarov/hunting-for-credentials-dumping-in-windows-environment - - https://dfironthemountain.wordpress.com/2018/12/06/locked-file-access-using-esentutl-exe/ -tags: - - attack.credential_access - - attack.t1003 - - car.2013-07-001 - - attack.t1003.002 - - attack.t1003.003 -logsource: - category: process_creation - product: windows -detection: - selection: - - Image|endswith: '\esentutl.exe' - CommandLine|contains: - - 'vss' - - ' /m ' - - ' /y ' - - CommandLine|contains: - - '\windows\ntds\ntds.dit' - - '\config\sam' - - '\config\security' - - '\config\system ' # space needed to avoid false positives with \config\systemprofile\ - - '\repair\sam' - - '\repair\system' - - '\repair\security' - - '\config\RegBack\sam' - - '\config\RegBack\system' - - '\config\RegBack\security' - condition: selection -falsepositives: - - Copying sensitive files for legitimate use (eg. backup) or forensic investigation by legitimate incident responder or forensic invetigator -level: high diff --git a/rules/windows/process_creation/win_susp_process_creations.yml b/rules/windows/process_creation/win_susp_process_creations.yml deleted file mode 100644 index 3d3254c3e..000000000 --- a/rules/windows/process_creation/win_susp_process_creations.yml +++ /dev/null @@ -1,74 +0,0 @@ -title: Suspicious Process Creation -id: 5f0f47a5-cb16-4dbe-9e31-e8d976d73de3 -description: Detects suspicious process starts on Windows systems based on keywords -status: experimental -references: - - https://www.swordshield.com/2015/07/getting-hashes-from-ntds-dit-file/ - - https://www.youtube.com/watch?v=H3t_kHQG1Js&feature=youtu.be&t=15m35s - - https://winscripting.blog/2017/05/12/first-entry-welcome-and-uac-bypass/ - - https://twitter.com/subTee/status/872244674609676288 - - https://docs.microsoft.com/en-us/windows-hardware/drivers/debugger/remote-tool-examples - - https://tyranidslair.blogspot.ca/2017/07/dg-on-windows-10-s-executing-arbitrary.html - - https://www.trustedsec.com/2017/07/new-tool-release-nps_payload/ - - https://subt0x10.blogspot.ca/2017/04/bypassing-application-whitelisting.html - - https://gist.github.com/subTee/7937a8ef07409715f15b84781e180c46#file-rat-bat - - https://twitter.com/vector_sec/status/896049052642533376 - - http://security-research.dyndns.org/pub/slides/FIRST-TC-2018/FIRST-TC-2018_Tom-Ueltschi_Sysmon_PUBLIC.pdf -author: Florian Roth, Daniil Yugoslavskiy, oscd.community (update) -date: 2018/01/01 -modified: 2019/11/01 -tags: - - car.2013-07-001 -logsource: - category: process_creation - product: windows -detection: - selection: - CommandLine: - - '* sekurlsa:*' - - net localgroup administrators * /add - - net group "Domain Admins" * /ADD /DOMAIN - - certutil.exe *-urlcache* http* - - certutil.exe *-urlcache* ftp* - - netsh advfirewall firewall *\AppData\\* - - attrib +S +H +R *\AppData\\* - - schtasks* /create *\AppData\\* - - schtasks* /sc minute* - - '*\Regasm.exe *\AppData\\*' - - '*\Regasm *\AppData\\*' - - '*\bitsadmin* /transfer*' - - '*\certutil.exe * -decode *' - - '*\certutil.exe * -decodehex *' - - '*\certutil.exe -ping *' - - icacls * /grant Everyone:F /T /C /Q - - '* wbadmin.exe delete catalog -quiet*' - - '*\wscript.exe *.jse' - - '*\wscript.exe *.js' - - '*\wscript.exe *.vba' - - '*\wscript.exe *.vbe' - - '*\cscript.exe *.jse' - - '*\cscript.exe *.js' - - '*\cscript.exe *.vba' - - '*\cscript.exe *.vbe' - - '*\fodhelper.exe' - - '*waitfor*/s*' - - '*waitfor*/si persist*' - - '*remote*/s*' - - '*remote*/c*' - - '*remote*/q*' - - '*AddInProcess*' - - '* /stext *' - - '* /scomma *' - - '* /stab *' - - '* /stabular *' - - '* /shtml *' - - '* /sverhtml *' - - '* /sxml *' - condition: selection -fields: - - ComputerName - - User - - CommandLine -falsepositives: - - False positives depend on scripts and administrative tools used in the monitored environment -level: medium diff --git a/tools/sigma/backends/sysmon.py b/tools/sigma/backends/sysmon.py index 3c8c74389..12171dc34 100644 --- a/tools/sigma/backends/sysmon.py +++ b/tools/sigma/backends/sysmon.py @@ -1,250 +1,244 @@ -# Output backends for sigmac -# Copyright 2020 SOC Prime - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. - -# You should have received a copy of the GNU Lesser General Public License -# along with this program. If not, see . - - -import re - -import sigma -from sigma.backends.base import SingleTextQueryBackend -from sigma.backends.mixins import MultiRuleOutputMixin - - -class SysmonConfigBackend(SingleTextQueryBackend, MultiRuleOutputMixin): - identifier = "sysmon" - active = True - andToken = " AND " - orToken = " OR " - notToken = "NOT " - subExpression = "(%s)" - config_required = False - INCLUDE = "include" - EXCLUDE = "exclude" - conditionDict = { - "startswith": "begin with", - "endswith": "end with", - } - - def __init__(self, *args, **kwargs): - self.table = None - self.logsource = None - self.allowedSource = { - "process_creation": "ProcessCreate" - } - self.eventidTagMapping = { - 1: "ProcessCreate", - 2: "FileCreateTime", - 3: "NetworkConnect", - 5: "ProcessTerminate", - 6: "DriverLoad", - 7: "ImageLoad", - 8: "CreateRemoteThread", - 9: "RawAccessRead", - 10: "ProcessAccess", - 11: "FileCreate", - 12: "RegistryEvent", - 13: "RegistryEvent", - 14: "RegistryEvent", - 15: "FileCreateStreamHash", - 17: "PipeEvent", - 18: "PipeEvent", - 19: "WmiEvent", - 20: "WmiEvent", - 21: "WmiEvent", - 22: "DNSQuery", - 23: "FileDelete" - } - self.allowedCondCombinations = { - 'single': [ - [4], - [1, 4], - [2, 4], - ], - 'multi': [ - [1, 2, 4], - ], - "exclude": [ - [1, 3, 4], - [2, 3, 4] - ], - # "multi-exclude": [ - # [1, 2, 3, 4] - # ] - } - return super().__init__(*args, **kwargs) - - def cleanValue(self, value): - val = re.sub("[*]", "", value) - return val - - def mapFiledValue(self, field, value): - condition = None - if "|" in field: - field, *pipes = field.split("|") - if len(pipes) == 1: - condition = pipes[0] - else: - raise NotImplementedError("not implemented condition") - if isinstance(value, list) and len(value) > 1: - condition = "contains any" - value = ";".join(value) - elif "*" in value: - if value.startswith("*") and value.endswith("*"): - condition = "contains" - elif value.startswith("*"): - condition = "end with" - elif value.endswith("*"): - condition = "begin with" - else: - condition = "contains" - - if condition: - field_str = '<{field} condition="{condition}">{value}'.format(field=field, - condition=condition, - value=self.cleanValue(value)) - else: - field_str = '<{field}>{value}'.format(field=field, value=self.cleanValue(value)) - - return field_str - - def createRule(self, selections): - fields_list = [] - table = None - for field, value in selections.items(): - if isinstance(value, list) and len(value) == 1: - value = value[0] - if field == "EventID": - table = self.eventidTagMapping[value] - else: - created_field_value = self.mapFiledValue(field, value) - fields_list.append(created_field_value) - fields_list_filtered = [item for item in fields_list if item] - if any(fields_list_filtered): - rule = '''\n\t\t\n\t\t\t{fields}\n\t\t'''.format(rule_name=self.rule_name, fields="\n\t\t\t".join(["{}".format(item) for item in fields_list_filtered])) - t = table if table else self.table - return rule, t - else: - return None, None - - def createRuleGroup(self, condition_objects, condition, match_type="include"): - rules = None - rules_selections = [item for item in condition_objects if item.type == 4] - if len(rules_selections) == 1: - rule, table = self.createRule(self.detection.get(rules_selections[0].matched)) - rules = {match_type: {table: rule}} - else: - if "or" in condition.lower(): - result = {} - for selection_object in rules_selections: - rule, table = self.createRule(self.detection.get(selection_object.matched)) - if result.get(table): - result[table].append(rule) - else: - result[table] = [rule] - result = {table_name: "\n\t\t".join(rules_list) for table_name, rules_list in result.items()} - rules = {match_type: result} - elif "and" in condition.lower(): - rules_dict = {} - for selection_object in rules_selections: - rules_dict.update(self.detection.get(selection_object.matched)) - rule, table = self.createRule(rules_dict) - rules = {match_type: {table: rule}} - if rules: - rules_result = [] - for match, tables in rules.items(): - for table, rules in tables.items(): - category_comment = '\n\n{}'.format(table, match, - "".join(rules)) - rules_result.append(category_comment) - return "".join(rules_result) - else: - raise - - def createMultiRuleGroup(self, conditions): - conditions_id = "".join([str(item.type) for item in conditions]) - or_index = conditions_id.index("2") - sorted_conditions = [conditions[:or_index], conditions[or_index+1:]] - if sorted_conditions: - result = "" - for rule_condition in sorted_conditions: - rule = self.createRuleGroup(condition_objects=rule_condition, condition=" ".join([item.matched for item in rule_condition])) - result += "{}\n".format(rule) - return result - else: - raise NotImplementedError("not implemented condition") - - def createExcludeRuleGroup(self, conditions): - conditions_id = "".join([str(item.type) for item in conditions]) - condition = self.detection.get("condition") - sorted_conditions = None - if "and not" in condition.lower(): - andnot_index = conditions_id.index("13") - sorted_conditions = [(conditions[:andnot_index], self.INCLUDE), ([item for item in conditions if item.type != 3], self.EXCLUDE)] - elif "or not" in condition.lower(): - ornot_index = conditions_id.index("23") - sorted_conditions = [(conditions[:ornot_index], self.INCLUDE), (conditions[ornot_index + 2:], self.EXCLUDE)] - if sorted_conditions: - result = "" - for rule_condition in sorted_conditions: - rule = self.createRuleGroup(condition_objects=rule_condition[0], condition=" ".join([item.matched for item in rule_condition[0]]), match_type=rule_condition[1]) - result += "{}\n".format(rule) - return result - - def createMultiExcludeRuleGroup(self, conditions): - return - - def checkRuleCondition(self, condtokens): - if len(condtokens) == 1: - conditions = [item for item in condtokens[0].tokens] - conditions_combination = list(set([item.type for item in conditions])) - for rule_type, combinations in self.allowedCondCombinations.items(): - for combination in combinations: - if sorted(conditions_combination) == sorted(combination): - return rule_type, conditions - else: - raise NotImplementedError("not implemented condition") - else: - raise NotImplementedError("not implemented condition") - - def createTableFromLogsource(self): - if self.logsource.get("product", "") != "windows": - raise TypeError( - "Not supported logsource. Should be product windows") - for item in self.logsource.values(): - if item.lower() in self.allowedSource.keys(): - self.table = self.allowedSource.get(item.lower()) - break - else: - self.table = "ProcessCreate" - - def finalize(self): - rulegroup_comment = '' - return "{}\n{}".format(rulegroup_comment, self.sysmon_rule) - - - def generate(self, sigmaparser): - title = sigmaparser.parsedyaml.get("title", "") - author = sigmaparser.parsedyaml.get("author", {}) - self.rule_name = "{} by {}".format(title, author) - self.detection = sigmaparser.parsedyaml.get("detection", {}) - self.logsource = sigmaparser.parsedyaml["logsource"] - self.createTableFromLogsource() - rule_type, conditions = self.checkRuleCondition(sigmaparser.condtoken) - if rule_type == "single": - self.sysmon_rule = self.createRuleGroup(conditions, self.detection.get("condition")) - elif rule_type == "multi": - self.sysmon_rule = self.createMultiRuleGroup(conditions) - elif rule_type == "exclude": - self.sysmon_rule = self.createExcludeRuleGroup(conditions) +import re + +import sigma +from sigma.backends.base import SingleTextQueryBackend +from sigma.backends.mixins import MultiRuleOutputMixin + +from .exceptions import NotSupportedError + + +class SysmonConfigBackend(SingleTextQueryBackend, MultiRuleOutputMixin): + identifier = "sysmon" + active = True + andToken = " AND " + orToken = " OR " + notToken = "NOT " + subExpression = "(%s)" + config_required = False + INCLUDE = "include" + EXCLUDE = "exclude" + conditionDict = { + "startswith": "begin with", + "endswith": "end with", + } + + def __init__(self, *args, **kwargs): + self.table = None + self.logsource = None + self.allowedSource = { + "process_creation": "ProcessCreate" + } + self.eventidTagMapping = { + 1: "ProcessCreate", + 4799: "ProcessCreate", + 2: "FileCreateTime", + 3: "NetworkConnect", + 5: "ProcessTerminate", + 6: "DriverLoad", + 7: "ImageLoad", + 8: "CreateRemoteThread", + 9: "RawAccessRead", + 10: "ProcessAccess", + 11: "FileCreate", + 12: "RegistryEvent", + 13: "RegistryEvent", + 14: "RegistryEvent", + 15: "FileCreateStreamHash", + 17: "PipeEvent", + 18: "PipeEvent", + 19: "WmiEvent", + 20: "WmiEvent", + 21: "WmiEvent", + 22: "DNSQuery", + 257: "DNSQuery", + 23: "FileDelete" + } + self.allowedCondCombinations = { + 'single': [ + [4], + [1, 4], + [2, 4], + ], + 'multi': [ + [1, 2, 4], + ], + "exclude": [ + [1, 3, 4], + [2, 3, 4] + ], + # "multi-exclude": [ + # [1, 2, 3, 4] + # ] + } + return super().__init__(*args, **kwargs) + + def cleanValue(self, value): + val = re.sub("[*]", "", value) + return val + + def mapFiledValue(self, field, value): + condition = None + if "|" in field: + field, *pipes = field.split("|") + if len(pipes) == 1: + condition = pipes[0] + else: + raise NotImplementedError("not implemented condition") + if isinstance(value, list) and len(value) > 1: + condition = "contains any" + value = ";".join(value) + elif "*" in value: + if value.startswith("*") and value.endswith("*"): + condition = "contains" + elif value.startswith("*"): + condition = "end with" + elif value.endswith("*"): + condition = "begin with" + else: + condition = "contains" + + if condition: + field_str = '<{field} condition="{condition}">{value}'.format(field=field, + condition=condition, + value=self.cleanValue(value)) + else: + field_str = '<{field}>{value}'.format(field=field, value=self.cleanValue(value)) + + return field_str + + def createRule(self, selections): + fields_list = [] + table = None + for field, value in selections.items(): + if isinstance(value, list) and len(value) == 1: + value = value[0] + if field == "EventID": + try: + table = self.eventidTagMapping[value] + except KeyError: + table = self.eventidTagMapping[1] + else: + created_field_value = self.mapFiledValue(field, value) + fields_list.append(created_field_value) + fields_list_filtered = [item for item in fields_list if item] + if any(fields_list_filtered): + rule = '''\n\t\t\n\t\t\t{fields}\n\t\t'''.format(rule_name=self.rule_name, fields="\n\t\t\t".join(["{}".format(item) for item in fields_list_filtered])) + t = table if table else self.table + return rule, t + else: + return None, None + + def createRuleGroup(self, condition_objects, condition, match_type="include"): + rules = None + rules_selections = [item for item in condition_objects if item.type == 4] + if len(rules_selections) == 1: + rule, table = self.createRule(self.detection.get(rules_selections[0].matched)) + rules = {match_type: {table: rule}} + else: + if "or" in condition.lower(): + result = {} + for selection_object in rules_selections: + rule, table = self.createRule(self.detection.get(selection_object.matched)) + if result.get(table): + result[table].append(rule) + else: + result[table] = [rule] + result = {table_name: "\n\t\t".join(rules_list) for table_name, rules_list in result.items()} + rules = {match_type: result} + elif "and" in condition.lower(): + rules_dict = {} + for selection_object in rules_selections: + rules_dict.update(self.detection.get(selection_object.matched)) + rule, table = self.createRule(rules_dict) + rules = {match_type: {table: rule}} + if rules: + rules_result = [] + for match, tables in rules.items(): + for table, rules in tables.items(): + category_comment = '\n\n{}'.format(table, match, + "".join(rules)) + rules_result.append(category_comment) + return "".join(rules_result) + else: + raise NotSupportedError("Couldn't create rule with current condition.") + + def createMultiRuleGroup(self, conditions): + conditions_id = "".join([str(item.type) for item in conditions]) + or_index = conditions_id.index("2") + sorted_conditions = [conditions[:or_index], conditions[or_index+1:]] + if sorted_conditions: + result = "" + for rule_condition in sorted_conditions: + rule = self.createRuleGroup(condition_objects=rule_condition, condition=" ".join([item.matched for item in rule_condition])) + result += "{}\n".format(rule) + return result + else: + raise NotSupportedError("Not implemented condition.") + + def createExcludeRuleGroup(self, conditions): + conditions_id = "".join([str(item.type) for item in conditions]) + condition = self.detection.get("condition") + sorted_conditions = None + if "and not" in condition.lower(): + andnot_index = conditions_id.index("13") + sorted_conditions = [(conditions[:andnot_index], self.INCLUDE), ([item for item in conditions if item.type != 3], self.EXCLUDE)] + elif "or not" in condition.lower(): + ornot_index = conditions_id.index("23") + sorted_conditions = [(conditions[:ornot_index], self.INCLUDE), (conditions[ornot_index + 2:], self.EXCLUDE)] + if sorted_conditions: + result = "" + for rule_condition in sorted_conditions: + rule = self.createRuleGroup(condition_objects=rule_condition[0], condition=" ".join([item.matched for item in rule_condition[0]]), match_type=rule_condition[1]) + result += "{}\n".format(rule) + return result + + def checkRuleCondition(self, condtokens): + if len(condtokens) == 1: + conditions = [item for item in condtokens[0].tokens] + conditions_combination = list(set([item.type for item in conditions])) + for rule_type, combinations in self.allowedCondCombinations.items(): + for combination in combinations: + if sorted(conditions_combination) == sorted(combination): + return rule_type, conditions + else: + raise NotSupportedError("Not supported condition.") + else: + raise NotSupportedError("Not supported condition.") + + def createTableFromLogsource(self): + if self.logsource.get("product", "") != "windows": + raise NotSupportedError( + "Not supported logsource. Should be product `windows`.") + for item in self.logsource.values(): + if item.lower() in self.allowedSource.keys(): + self.table = self.allowedSource.get(item.lower()) + break + else: + self.table = "ProcessCreate" + + def checkDetection(self): + for selection_name, value in self.detection.items(): + if isinstance(value, list): + raise NotSupportedError("Keywords are not supported in sysmon backend.") + + + def generate(self, sigmaparser): + sysmon_rule = None + title = sigmaparser.parsedyaml.get("title", "") + author = sigmaparser.parsedyaml.get("author", {}) + self.rule_name = "{} by {}".format(title, author) + self.detection = sigmaparser.parsedyaml.get("detection", {}) + self.checkDetection() + self.logsource = sigmaparser.parsedyaml["logsource"] + self.createTableFromLogsource() + rule_type, conditions = self.checkRuleCondition(sigmaparser.condtoken) + if rule_type == "single": + sysmon_rule = self.createRuleGroup(conditions, self.detection.get("condition")) + elif rule_type == "multi": + sysmon_rule = self.createMultiRuleGroup(conditions) + elif rule_type == "exclude": + sysmon_rule = self.createExcludeRuleGroup(conditions) + + if sysmon_rule: + rulegroup_comment = '' + return "{}\n{}".format(rulegroup_comment, sysmon_rule) \ No newline at end of file From 64035fd799e563e32de8975f0ecc1d6e867ceeb6 Mon Sep 17 00:00:00 2001 From: snake-jump Date: Thu, 10 Sep 2020 17:12:12 +0200 Subject: [PATCH 03/11] initial commit for Netwitness-EPL backend --- tools/config/netwitness-epl.yml | 92 +++++++++++++++ tools/sigma/backends/netwitness-epl.py | 155 +++++++++++++++++++++++++ 2 files changed, 247 insertions(+) create mode 100644 tools/config/netwitness-epl.yml create mode 100644 tools/sigma/backends/netwitness-epl.py diff --git a/tools/config/netwitness-epl.yml b/tools/config/netwitness-epl.yml new file mode 100644 index 000000000..1709092b5 --- /dev/null +++ b/tools/config/netwitness-epl.yml @@ -0,0 +1,92 @@ +title: NetWitness +order: 20 +backends: + - netwitness-epl +logsources: + linux: + product: linux + conditions: + device.class: rhlinux + linux-sshd: + product: linux + service: sshd + conditions: + device.class: rhlinux + client: sshd + linux-auth: + product: linux + service: auth + conditions: + device.class: rhlinux + linux-clamav: + product: linux + service: clamav + conditions: + device.class: rhlinux + windows-sys: + product: windows + service: sysmon + conditions: + device.type: winevent_nic + event.source: microsoft-windows-security-auditing + windows-power: + product: windows + service: powershell + conditions: + device.type: winevent_nic + windows-dhcp: + product: windows + service: dhcp + conditions: + device.type: winevent_nic + event.source: microsoft-windows-dhcp-server + windows-sec: + product: windows + service: security + conditions: + device.type: winevent_nic + event.source: microsoft-windows-security-auditing + windows-system: + product: windows + service: system + conditions: + device.type: winevent_nic +fieldmappings: + dst: + - ip.dst + dst_ip: + - ip.dst + src: + - ip.src + src_ip: + - ip.src + DestinationPort: + - ip.dstport + EventID: + - reference.id + NewProcessName: + - process + LogonType: + - logon.type + AccountName: + - user.dst + c-uri-extension: + - extension + c-useragent: + - user.agent + r-dns: + - alias.host + DestinationHostname: + - alias.host + cs-host: + - alias.host + c-uri-query: + - web.page + c-uri: + - web.page + cs-method: + - action + cs-cookie: + - web.cookie + SubjectUserName: + - user.dst diff --git a/tools/sigma/backends/netwitness-epl.py b/tools/sigma/backends/netwitness-epl.py new file mode 100644 index 000000000..17862544b --- /dev/null +++ b/tools/sigma/backends/netwitness-epl.py @@ -0,0 +1,155 @@ +# NetWitness EPL output backend for sigmac +# Copyright 2019 Tarik BOUDJEMAA (@snake-jump) +# Inspired from John Tuckner (@tuckner) NetWitness output backend for sigmac + +# NetWitness EPL backend for sigmac uses netwitness-epl.yml config file + + +# RSA Alerts are generated by Event Processing Language (EPL) , that uses Esper Engine (https://www.espertech.com/esper/) +# For more details see :https://community.rsa.com/docs/DOC-110246 and https://community.rsa.com/docs/DOC-80068 + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. + +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + + +import re +import sigma +from .base import SingleTextQueryBackend +from .mixins import MultiRuleOutputMixin +import sqlparse + +template=""" +module_XXXXX; +@Name('RuleName') +@RSAAlert(oneInSeconds=0) +SELECT * FROM Event( + +EXPRESSION +);""" + +class NetWitnessEplBackend(SingleTextQueryBackend): + """Converts Sigma rule into RSA NetWitness EPL . Contributed by @snake-jump""" + identifier = "netwitness-epl" + config_required = False + default_config = ["netwitness-epl"] + active = True + reEscape = re.compile('(")') + ##reEscape = re.compile("([\\|()\[\]{}.^$+])") + reClear = None + andToken = " AND " + orToken = " OR " + notToken = "NOT" + subExpression = "(%s)" + listExpression = "(%s)" + listSeparator = ", " + valueExpression = "\'%s\'" + keyExpression = "%s" + nullExpression = "%s exists" + notNullExpression = "%s exists" + mapExpression = "(%s=%s)" + mapListsSpecialHandling = True + + def generateMapItemNode(self, node): + key, value = node + if type(key) != int: + key = key.replace(".","_") ## replace . by _ in meta name (RSA EPL) + key = key.lower() + if self.mapListsSpecialHandling == False and type(value) in (str, int, list) or self.mapListsSpecialHandling == True and type(value) in (str, int): + if type(value) == str and "*" in value[1:-1]: + value = re.sub('([".^$]|\\\\(?![*?]))', '\\\\\g<1>', value) + value = re.sub('\\*', '.*', value) + value = re.sub('\\?', '.', value) + return "(%s REGEXP %s)" %(key, self.generateValueNode(value)) + elif type(value) == str and "*" in value: + value = re.sub("(\*\\\\)|(\*)", "", value) + value = self.generateValueNode("%"+value+"%") # add "%" to construct the like expression ex: process like %psexesvc% + return "(%s LIKE %s)" % (key,value) + elif type(value) in (str, int): + return self.mapExpression % (key, self.generateValueNode(value)) + else: + return self.mapExpression % (key, self.generateNode(value)) + elif type(value) == list: + return self.generateMapItemListNode(key, value) + elif value is None: + return self.nullExpression % (key, ) + + else: + raise TypeError("Backend does not support map values of type " + str(type(value))) + + def generateMapItemListNode(self, key, value): + equallist = list() + containlist = list() + regexlist = list() + for item in value: + if type(item) == str and "*" in item[1:-1]: + item = re.sub('([".^$]|\\\\(?![*?]))', '\\\\\g<1>', item) + item = re.sub('\\*', '.*', item) + item = re.sub('\\?', '.', item) + regexlist.append(self.generateValueNode(item)) + elif type(item) == str and (item.endswith("*") or item.startswith("*")): + item_temp=item + item = re.sub("(\*\\\\)|(\*)", "", item) + if item_temp.endswith("*") and item_temp.startswith("*"): # pattern begins with "*" and ends with "*" + containlist.append(self.generateValueNode('%'+item+'%')) # add "%" to construct the like expression ex: process like %psexesvc% + elif item_temp.startswith("*"): # pattern don't end with "*" + containlist.append(self.generateValueNode('%'+item)) + else: # item_temp.endswith("*") pattern don't begin with "*" + containlist.append(self.generateValueNode(item+'%')) + else: + equallist.append(self.generateValueNode(item)) + fmtitems = list() + if equallist: + if len(equallist) == 1: + fmtitems.append("%s = %s" % (key, ", ".join(equallist))) + else: + # add "(" and ")" to the first and the last item from the list to have meta_key IN ('value1','value2') + equallist[0]=("("+equallist[0]) + equallist[-1]=(equallist[-1]+")") + fmtitems.append("%s IN %s" % (key, ", ".join(equallist))) + + if containlist: + fmtitems.append("%s LIKE %s" % (key, (" OR "+key+" LIKE ").join(containlist))) + if regexlist: + fmtitems.append("%s REGEXP %s" % (key, "|".join(regexlist))) + fmtquery = "("+" OR ".join(filter(None, fmtitems)) + # delete the " ' " from the begin or the end of the each regex pattern ex : '.*('patern1'|'patern2').*' --> '.*(patern1|patern2).*' + fmtquery = re.sub('\'\.\*\(\'','\'.*(',fmtquery) + fmtquery = re.sub('\'\)\.\*\'',').*\'',fmtquery) + fmtquery = re.sub('\'\|\'','|',fmtquery) + fmtquery = fmtquery+')' + return fmtquery + + def generateValueNode(self, node): + return self.valueExpression % (str(node)) + + def generate(self, sigmaparser): + """Method is called for each sigma rule and receives the parsed rule (SigmaParser)""" + for k in sigmaparser.parsedyaml["detection"].keys(): + if k.startswith('keyword'): + raise NotImplementedError("Backend does not support keywords") + for parsed in sigmaparser.condparsed: + query = self.generateQuery(parsed, sigmaparser) + query = sqlparse.format(query, reindent=True, keyword_case='upper') + query=query.replace('INDEX','`index`') # index is reserved keyword in Esper and must be escaped + query=template.replace('EXPRESSION', query) + try: + query=query.replace('RuleName', sigmaparser.parsedyaml["title"].replace(" ","")) # add rule name + query=query.replace('module_XXXXX', "module "+sigmaparser.parsedyaml["title"].replace(" ","")) # add rule name + except: + print("Error when replacing RuleName by Title from yaml") + pass + return query + + def generateQuery(self, parsed, sigmaparser): + result = self.generateNode(parsed.parsedSearch) + return result \ No newline at end of file From e74846b76752aaa3634674de6bfedb435278eeef Mon Sep 17 00:00:00 2001 From: snake-jump Date: Thu, 10 Sep 2020 18:09:15 +0200 Subject: [PATCH 04/11] modify comment --- tools/sigma/backends/netwitness-epl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/sigma/backends/netwitness-epl.py b/tools/sigma/backends/netwitness-epl.py index 17862544b..259844afc 100644 --- a/tools/sigma/backends/netwitness-epl.py +++ b/tools/sigma/backends/netwitness-epl.py @@ -122,7 +122,7 @@ class NetWitnessEplBackend(SingleTextQueryBackend): if regexlist: fmtitems.append("%s REGEXP %s" % (key, "|".join(regexlist))) fmtquery = "("+" OR ".join(filter(None, fmtitems)) - # delete the " ' " from the begin or the end of the each regex pattern ex : '.*('patern1'|'patern2').*' --> '.*(patern1|patern2).*' + # Delete the " ' " from the begin or the end of each regex pattern ex : '.*('patern1'|'patern2').*' --> '.*(patern1|patern2).*' fmtquery = re.sub('\'\.\*\(\'','\'.*(',fmtquery) fmtquery = re.sub('\'\)\.\*\'',').*\'',fmtquery) fmtquery = re.sub('\'\|\'','|',fmtquery) From 09f25cf992619a2c5617c5bbcc1b0167ab0d321d Mon Sep 17 00:00:00 2001 From: snake-jump Date: Thu, 10 Sep 2020 19:05:55 +0200 Subject: [PATCH 05/11] delete sqlparse module usage --- tools/sigma/backends/netwitness-epl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/sigma/backends/netwitness-epl.py b/tools/sigma/backends/netwitness-epl.py index 259844afc..4715ba348 100644 --- a/tools/sigma/backends/netwitness-epl.py +++ b/tools/sigma/backends/netwitness-epl.py @@ -26,7 +26,7 @@ import re import sigma from .base import SingleTextQueryBackend from .mixins import MultiRuleOutputMixin -import sqlparse +#import sqlparse template=""" module_XXXXX; @@ -139,7 +139,7 @@ class NetWitnessEplBackend(SingleTextQueryBackend): raise NotImplementedError("Backend does not support keywords") for parsed in sigmaparser.condparsed: query = self.generateQuery(parsed, sigmaparser) - query = sqlparse.format(query, reindent=True, keyword_case='upper') + #query = sqlparse.format(query, reindent=True, keyword_case='upper') query=query.replace('INDEX','`index`') # index is reserved keyword in Esper and must be escaped query=template.replace('EXPRESSION', query) try: From 1f50e0af3504633d0c1ec9317ba215d04701a7d5 Mon Sep 17 00:00:00 2001 From: Scott Dermott Date: Fri, 11 Sep 2020 16:06:51 +0100 Subject: [PATCH 06/11] + Adding exclusion for Azure AD Sync (MSOL_xxxxxxxx) AD Connect on premise AD accounts to Azure AD. The replication process is completed under the context of the 'MSOL_xxxxxxxx' user account. The AD Connect application is installed on a member server (i.e. not on a DC). https://techcommunity.microsoft.com/t5/azure-advanced-threat-protection/ad-connect-msol-user-suspected-dcsync-attack/m-p/788028 --- rules/windows/builtin/win_dcsync.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/rules/windows/builtin/win_dcsync.yml b/rules/windows/builtin/win_dcsync.yml index 1181f0e18..e58e054a7 100644 --- a/rules/windows/builtin/win_dcsync.yml +++ b/rules/windows/builtin/win_dcsync.yml @@ -3,8 +3,8 @@ id: 611eab06-a145-4dfa-a295-3ccc5c20f59a description: Detects Mimikatz DC sync security events status: experimental date: 2018/06/03 -modified: 2019/10/08 -author: Benjamin Delpy, Florian Roth +modified: 2020/09/11 +author: Benjamin Delpy, Florian Roth, Scott Dermott references: - https://twitter.com/gentilkiwi/status/1003236624925413376 - https://gist.github.com/gentilkiwi/dcc132457408cf11ad2061340dcb53c2 @@ -28,6 +28,7 @@ detection: SubjectUserName: - 'NT AUTHORITY*' - '*$' + - 'MSOL_*' condition: selection and not filter1 and not filter2 falsepositives: - Valid DC Sync that is not covered by the filters; please report From 1fc202fe5d5bf2dea366e90262ff2bf5e9bf217b Mon Sep 17 00:00:00 2001 From: Yugoslavskiy Daniil Date: Sun, 13 Sep 2020 15:46:45 +0200 Subject: [PATCH 07/11] fix typos, update tags --- rules/windows/builtin/win_disable_event_logging.yml | 2 +- rules/windows/other/win_defender_psexec_wmi_asr.yml | 3 +-- rules/windows/process_access/sysmon_invoke_phantom.yml | 2 +- rules/windows/process_creation/win_etw_trace_evasion.yml | 2 +- rules/windows/process_creation/win_susp_covenant.yml | 2 ++ rules/windows/registry_event/sysmon_susp_lsass_dll_load.yml | 2 +- 6 files changed, 7 insertions(+), 6 deletions(-) diff --git a/rules/windows/builtin/win_disable_event_logging.yml b/rules/windows/builtin/win_disable_event_logging.yml index da1f92fe0..52ef34e3f 100644 --- a/rules/windows/builtin/win_disable_event_logging.yml +++ b/rules/windows/builtin/win_disable_event_logging.yml @@ -6,7 +6,7 @@ references: tags: - attack.defense_evasion - attack.t1054 # an old one - - attack.t1562.006 + - attack.t1562.002 author: '@neu5ron' date: 2017/11/19 logsource: diff --git a/rules/windows/other/win_defender_psexec_wmi_asr.yml b/rules/windows/other/win_defender_psexec_wmi_asr.yml index 850023895..6761ba143 100644 --- a/rules/windows/other/win_defender_psexec_wmi_asr.yml +++ b/rules/windows/other/win_defender_psexec_wmi_asr.yml @@ -10,9 +10,8 @@ date: 2020/07/14 tags: - attack.execution - attack.lateral_movement - - attack.t1570 - attack.t1047 - - attack.t1569 + - attack.t1035 # an old one - attack.t1569.002 logsource: product: windows_defender diff --git a/rules/windows/process_access/sysmon_invoke_phantom.yml b/rules/windows/process_access/sysmon_invoke_phantom.yml index a389a9a56..bbcf116ae 100755 --- a/rules/windows/process_access/sysmon_invoke_phantom.yml +++ b/rules/windows/process_access/sysmon_invoke_phantom.yml @@ -10,7 +10,7 @@ references: - https://twitter.com/timbmsft/status/900724491076214784 tags: - attack.defense_evasion - - attck.t1562.002 + - attack.t1562.002 - attack.t1089 # an old one logsource: category: process_access diff --git a/rules/windows/process_creation/win_etw_trace_evasion.yml b/rules/windows/process_creation/win_etw_trace_evasion.yml index 84c4fa7b8..71bb05e64 100644 --- a/rules/windows/process_creation/win_etw_trace_evasion.yml +++ b/rules/windows/process_creation/win_etw_trace_evasion.yml @@ -11,7 +11,7 @@ date: 2019/03/22 tags: - attack.defense_evasion - attack.t1070 - - attack.t1562 + - attack.t1562.006 - car.2016-04-002 level: high logsource: diff --git a/rules/windows/process_creation/win_susp_covenant.yml b/rules/windows/process_creation/win_susp_covenant.yml index 40fa8950f..d2440ff5c 100644 --- a/rules/windows/process_creation/win_susp_covenant.yml +++ b/rules/windows/process_creation/win_susp_covenant.yml @@ -8,7 +8,9 @@ author: Florian Roth date: 2020/06/04 tags: - attack.execution + - attack.defense_evasion - attack.t1059.001 + - attack.t1564.003 - attack.t1086 # an old one logsource: category: process_creation diff --git a/rules/windows/registry_event/sysmon_susp_lsass_dll_load.yml b/rules/windows/registry_event/sysmon_susp_lsass_dll_load.yml index bf440234b..e7ff37013 100644 --- a/rules/windows/registry_event/sysmon_susp_lsass_dll_load.yml +++ b/rules/windows/registry_event/sysmon_susp_lsass_dll_load.yml @@ -19,9 +19,9 @@ detection: condition: selection tags: - attack.execution + - attack.persistence - attack.t1177 # an old one - attack.t1547.008 falsepositives: - Unknown level: high - From 531557465c33ea2839f0842ea151c3a3424349d3 Mon Sep 17 00:00:00 2001 From: snake-jump Date: Mon, 14 Sep 2020 16:00:03 +0200 Subject: [PATCH 08/11] delete raise exception in case of sigma key is keyword(s) --- tools/sigma/backends/netwitness-epl.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tools/sigma/backends/netwitness-epl.py b/tools/sigma/backends/netwitness-epl.py index 4715ba348..0e2d8fbdf 100644 --- a/tools/sigma/backends/netwitness-epl.py +++ b/tools/sigma/backends/netwitness-epl.py @@ -26,7 +26,6 @@ import re import sigma from .base import SingleTextQueryBackend from .mixins import MultiRuleOutputMixin -#import sqlparse template=""" module_XXXXX; @@ -134,12 +133,8 @@ class NetWitnessEplBackend(SingleTextQueryBackend): def generate(self, sigmaparser): """Method is called for each sigma rule and receives the parsed rule (SigmaParser)""" - for k in sigmaparser.parsedyaml["detection"].keys(): - if k.startswith('keyword'): - raise NotImplementedError("Backend does not support keywords") for parsed in sigmaparser.condparsed: query = self.generateQuery(parsed, sigmaparser) - #query = sqlparse.format(query, reindent=True, keyword_case='upper') query=query.replace('INDEX','`index`') # index is reserved keyword in Esper and must be escaped query=template.replace('EXPRESSION', query) try: @@ -152,4 +147,4 @@ class NetWitnessEplBackend(SingleTextQueryBackend): def generateQuery(self, parsed, sigmaparser): result = self.generateNode(parsed.parsedSearch) - return result \ No newline at end of file + return result From 5119f887c83f276cac45afd94d941784744af587 Mon Sep 17 00:00:00 2001 From: snake-jump Date: Mon, 14 Sep 2020 22:04:47 +0200 Subject: [PATCH 09/11] add Regular expression support Add Regular expression support for netwitness-epl backend --- tools/sigma/backends/netwitness-epl.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/tools/sigma/backends/netwitness-epl.py b/tools/sigma/backends/netwitness-epl.py index 0e2d8fbdf..e580b259c 100644 --- a/tools/sigma/backends/netwitness-epl.py +++ b/tools/sigma/backends/netwitness-epl.py @@ -26,6 +26,8 @@ import re import sigma from .base import SingleTextQueryBackend from .mixins import MultiRuleOutputMixin +from sigma.parser.modifiers.base import SigmaTypeModifier +from sigma.parser.modifiers.type import SigmaRegularExpressionModifier template=""" module_XXXXX; @@ -40,10 +42,10 @@ class NetWitnessEplBackend(SingleTextQueryBackend): """Converts Sigma rule into RSA NetWitness EPL . Contributed by @snake-jump""" identifier = "netwitness-epl" config_required = False - default_config = ["netwitness-epl"] + default_config = ["sysmon","netwitness-epl"] active = True reEscape = re.compile('(")') - ##reEscape = re.compile("([\\|()\[\]{}.^$+])") + #reEscape = re.compile("([\\|()\[\]{}.^$+])") reClear = None andToken = " AND " orToken = " OR " @@ -82,6 +84,17 @@ class NetWitnessEplBackend(SingleTextQueryBackend): elif value is None: return self.nullExpression % (key, ) + elif type(value) == SigmaRegularExpressionModifier: ## if value is regex + regex = str(value) + ## in RSA netwitness EPL regex each backslash must be escaped by backslash + ## ex : c:\temp\ to regex -> c:\\temp\\ to RSA EPL regex --> c:\\\\temp\\\\ + regex = regex.replace("\\","\\\\") + # Regular Expressions have to match the full value in RSA Netwitness EPL + if not (regex.startswith('^') or regex.startswith('.*')): + regex = '.*' + regex + if not (regex.endswith('$') or regex.endswith('.*')): + regex = regex + '.*' + return "(%s REGEXP %s)" %(key, self.generateValueNode(regex)) else: raise TypeError("Backend does not support map values of type " + str(type(value))) From 64961c6d42e7f788f08632e1a3ac5f25069ccdfe Mon Sep 17 00:00:00 2001 From: Thomas Patzke Date: Tue, 15 Sep 2020 09:06:02 +0200 Subject: [PATCH 10/11] Added test --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index a9a5a0f6f..b8703f5da 100644 --- a/Makefile +++ b/Makefile @@ -57,6 +57,7 @@ test-sigmac: $(COVERAGE) run -a --include=$(COVSCOPE) tools/sigmac -rvdI -t carbonblack -c tools/config/carbon-black.yml rules/ > /dev/null $(COVERAGE) run -a --include=$(COVSCOPE) tools/sigmac -rvdI -t qualys -c tools/config/qualys.yml rules/ > /dev/null $(COVERAGE) run -a --include=$(COVSCOPE) tools/sigmac -rvdI -t netwitness -c tools/config/netwitness.yml rules/ > /dev/null + $(COVERAGE) run -a --include=$(COVSCOPE) tools/sigmac -rvdI -t netwitness-epl -c netwitness-epl rules/ > /dev/null $(COVERAGE) run -a --include=$(COVSCOPE) tools/sigmac -rvdI -t sumologic -O rulecomment -c tools/config/sumologic.yml rules/ > /dev/null $(COVERAGE) run -a --include=$(COVSCOPE) tools/sigmac -rvdI -t humio -O rulecomment -c tools/config/humio.yml rules/ > /dev/null $(COVERAGE) run -a --include=$(COVSCOPE) tools/sigmac -rvdI -t crowdstrike -O rulecomment -c tools/config/crowdstrike.yml rules/ > /dev/null From b0ccf442437f60d34910ebce06fafb73c2c6e16f Mon Sep 17 00:00:00 2001 From: Thomas Patzke Date: Tue, 15 Sep 2020 12:20:46 +0200 Subject: [PATCH 11/11] Added test --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index b8703f5da..8439b5dd7 100644 --- a/Makefile +++ b/Makefile @@ -65,6 +65,7 @@ test-sigmac: $(COVERAGE) run -a --include=$(COVSCOPE) tools/sigmac -rvdI -t sqlite -c sysmon rules/ > /dev/null $(COVERAGE) run -a --include=$(COVSCOPE) tools/sigmac -rvdI -t csharp -c sysmon rules/ > /dev/null $(COVERAGE) run -a --include=$(COVSCOPE) tools/sigmac -rvdI -t logiq -c sysmon rules/ > /dev/null + $(COVERAGE) run -a --include=$(COVSCOPE) tools/sigmac -rvdI -t sysmon -c sysmon -rvd rules/windows/driver_load rules/windows/file_event rules/windows/image_load rules/windows/network_connection rules/windows/process_access rules/windows/process_creation rules/windows/registry_event rules/windows/sysmon > /dev/null $(COVERAGE) run -a --include=$(COVSCOPE) tools/sigmac -rvdI -t splunk -c tools/config/splunk-windows-index.yml -f 'level>=high,level<=critical,status=stable,logsource=windows,tag=attack.execution' rules/ > /dev/null ! $(COVERAGE) run -a --include=$(COVSCOPE) tools/sigmac -rvdI -t splunk -c tools/config/splunk-windows-index.yml -f 'level>=high,level<=critical,status=xstable,logsource=windows' rules/ > /dev/null ! $(COVERAGE) run -a --include=$(COVSCOPE) tools/sigmac -rvdI -t splunk -c tools/config/splunk-windows-index.yml -f 'level>=high,level<=xcritical,status=stable,logsource=windows' rules/ > /dev/null