diff --git a/Makefile b/Makefile index dc5b7a5a0..98d5807e2 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: test test-yaml test-sigmac + .PHONY: test test-yaml test-sigmac test: test-yaml test-sigmac test-yaml: @@ -7,6 +7,8 @@ test-yaml: test-sigmac: tools/sigmac.py -l tools/sigmac.py -rvdI -t es-qs rules/ + tools/sigmac.py -rvdI -t kibana rules/ + tools/sigmac.py -rvdI -t xpack-watcher rules/ tools/sigmac.py -rvdI -t splunk rules/ tools/sigmac.py -rvdI -t logpoint rules/ tools/sigmac.py -rvdI -t fieldlist rules/ diff --git a/rules/application/appframework_django_exceptins.yml b/rules/application/appframework_django_exceptions.yml similarity index 100% rename from rules/application/appframework_django_exceptins.yml rename to rules/application/appframework_django_exceptions.yml diff --git a/rules/proxy/proxy_ua_apt.yml b/rules/proxy/proxy_ua_apt.yml index c15d8253a..7c2f45317 100644 --- a/rules/proxy/proxy_ua_apt.yml +++ b/rules/proxy/proxy_ua_apt.yml @@ -24,6 +24,7 @@ detection: - 'Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-EN; rv:1.7.12) Gecko/20100719 Firefox/1.0.7' # Unit78020 Malware - 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.2.13) Firefox/3.6.13 GTB7.1' # Winnti related - 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)' # Winnti related + - 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NETCLR 2.0.50727)' # APT17 condition: selection fields: - ClientIP diff --git a/rules/web/web_multiple_suspicious_resp_codes_single_source.yml b/rules/web/web_multiple_suspicious_resp_codes_single_source.yml index b2993d94a..935b406ae 100644 --- a/rules/web/web_multiple_suspicious_resp_codes_single_source.yml +++ b/rules/web/web_multiple_suspicious_resp_codes_single_source.yml @@ -14,6 +14,7 @@ detection: condition: selection | count() by clientip > 10 fields: - client_ip + - vhost - url - response falsepositives: diff --git a/rules/web/web_webshell_keyword.yml b/rules/web/web_webshell_keyword.yml index b2d8988ac..f6b453cc6 100644 --- a/rules/web/web_webshell_keyword.yml +++ b/rules/web/web_webshell_keyword.yml @@ -11,6 +11,7 @@ detection: condition: keywords fields: - client_ip + - vhost - url - response falsepositives: diff --git a/tools/backends.py b/tools/backends.py index a84caf402..45943ec8e 100644 --- a/tools/backends.py +++ b/tools/backends.py @@ -1,5 +1,6 @@ # Output backends for sigmac +import sys import json import re import sigma @@ -17,25 +18,128 @@ def getBackend(name): except KeyError as e: raise LookupError("Backend not found") from e -### Generic base classes +class BackendOptions(dict): + """Object contains all options that should be passed to the backend from command line (or other user interfaces)""" + def __init__(self, options): + """ + Receives the argparser result from the backend option paramater value list (nargs=*) and builds the dict from it. There are two option types: + + * key=value: self{key} = value + * key: self{key} = True + """ + if options == None: + return + for option in options: + parsed = option.split("=", 1) + try: + self[parsed[0]] = parsed[1] + except IndexError: + self[parsed[0]] = True + +### Output classes +class SingleOutput: + """ + Single file output + + By default, this opens the given file or stdin and passes everything into this. + """ + def __init__(self, filename=None): + if type(filename) == str: + self.fd = open(filename, "w") + else: + self.fd = sys.stdout + + def print(self, *args, **kwargs): + print(*args, file=self.fd, **kwargs) + + def close(self): + self.fd.close() + +class MultiOutput: + """ + Multiple file output + + Prepares multiple SingleOutput instances with basename + suffix as file names, on for each suffix. + The switch() method is used to switch between these outputs. + + This class must be inherited and suffixes must be a dict as follows: file id -> suffix + """ + suffixes = None + + def __init__(self, basename): + """Initializes all outputs with basename and corresponding suffix as SingleOutput object.""" + if suffixes == None: + raise NotImplementedError("OutputMulti must be derived, at least suffixes must be set") + if type(basename) != str: + raise TypeError("OutputMulti constructor basename parameter must be string") + + self.outputs = dict() + self.output = None + for name, suffix in self.suffixes: + self.outputs[name] = SingleOutput(basename + suffix) + + def select(self, name): + """Select an output as current output""" + self.output = self.outputs[name] + + def print(self, *args, **kwargs): + self.output.print(*args, **kwargs) + + def close(self): + for out in self.outputs: + out.close() + +class StringOutput(SingleOutput): + """Collect input silently and return resulting string.""" + def __init__(self, filename=None): + self.out = "" + + def print(self, *args, **kwargs): + try: + del kwargs['file'] + except KeyError: + pass + print(*args, file=self, **kwargs) + + def write(self, s): + self.out += s + + def result(self): + return self.out + + def close(self): + pass + +### Generic backend base classes and mixins class BaseBackend: """Base class for all backends""" identifier = "base" active = False - index_field = None # field name that is used to address indices + index_field = None # field name that is used to address indices + output_class = None # one of the above output classes + file_list = None - def __init__(self, sigmaconfig): + def __init__(self, sigmaconfig, backend_options=None, filename=None): + """ + Initialize backend. This gets a sigmaconfig object, which is notified about the used backend class by + passing the object instance to it. Further, output files are initialized by the output class defined in output_class. + """ + super().__init__() if not isinstance(sigmaconfig, (sigma.SigmaConfiguration, None)): raise TypeError("SigmaConfiguration object expected") + self.options = backend_options self.sigmaconfig = sigmaconfig self.sigmaconfig.set_backend(self) + self.output = self.output_class(filename) - def generate(self, parsed): - result = self.generateNode(parsed.parsedSearch) - if parsed.parsedAgg: - result += self.generateAggregation(parsed.parsedAgg) - return result + def generate(self, sigmaparser): + """Method is called for each sigma rule and receives the parsed rule (SigmaParser)""" + for parsed in sigmaparser.condparsed: + result = self.generateNode(parsed.parsedSearch) + if parsed.parsedAgg: + result += self.generateAggregation(parsed.parsedAgg) + self.output.print(result) def generateNode(self, node): if type(node) == sigma.ConditionAND: @@ -79,10 +183,18 @@ class BaseBackend: def generateAggregation(self, agg): raise NotImplementedError("Aggregations not implemented for this backend") + def finalize(self): + """ + Is called after the last file was processed with generate(). The right place if this backend is not intended to + look isolated at each rule, but generates an output which incorporates multiple rules, e.g. dashboards. + """ + pass + class SingleTextQueryBackend(BaseBackend): """Base class for backends that generate one text-based expression from a Sigma rule""" identifier = "base-textquery" active = False + output_class = SingleOutput # the following class variables define the generation and behavior of queries from a parse tree some are prefilled with default values that are quite usual reEscape = None # match characters that must be quoted @@ -138,6 +250,32 @@ class SingleTextQueryBackend(BaseBackend): def generateValueNode(self, node): return self.valueExpression % (self.cleanValue(str(node))) +class MultiRuleOutputMixin: + """Mixin with common for multi-rule outputs""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.rulenames = set() + + def getRuleName(self, sigmaparser): + """ + Generate a rule name from the title of the Sigma rule with following properties: + + * Spaces are replaced with - + * Unique name by addition of a counter if generated name already in usage + + Generated names are tracked by the Mixin. + + """ + rulename = sigmaparser.parsedyaml["title"].replace(" ", "-") + if rulename in self.rulenames: # add counter if name collides + cnt = 2 + while "%s-%d" % (rulename, cnt) in self.rulenames: + cnt += 1 + rulename = "%s-%d" % (rulename, cnt) + self.rulenames.add(rulename) + + return rulename + ### Backends for specific SIEMs class ElasticsearchQuerystringBackend(SingleTextQueryBackend): @@ -157,15 +295,184 @@ class ElasticsearchQuerystringBackend(SingleTextQueryBackend): mapExpression = "%s:%s" mapListsSpecialHandling = False -class ElasticsearchDSLBackend(BaseBackend): - """Converts Sigma rule into Elasticsearch DSL query (JSON).""" - identifier = "es-dsl" - active = False - -class KibanaBackend(ElasticsearchDSLBackend): - """Converts Sigma rule into Kibana JSON Configurations.""" +class KibanaBackend(ElasticsearchQuerystringBackend, MultiRuleOutputMixin): + """Converts Sigma rule into Kibana JSON Configuration files (searches only).""" identifier = "kibana" - active = False + active = True + output_class = SingleOutput + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.kibanaconf = list() + + def generate(self, sigmaparser): + rulename = self.getRuleName(sigmaparser) + description = sigmaparser.parsedyaml.setdefault("description", "") + + columns = list() + try: + for field in sigmaparser.parsedyaml["fields"]: + mapped = sigmaparser.config.get_fieldmapping(field).resolve_fieldname(field) + if type(mapped) == str: + columns.append(mapped) + elif type(mapped) == list: + columns.extend(mapped) + else: + raise TypeError("Field mapping must return string or list") + except KeyError: # no 'fields' attribute + pass + + indices = sigmaparser.get_logsource().index + if len(indices) == 0: + indices = ["logstash-*"] + + for parsed in sigmaparser.condparsed: + result = self.generateNode(parsed.parsedSearch) + + for index in indices: + final_rulename = rulename + if len(indices) > 1: # add index names if rule must be replicated because of ambigiuous index patterns + final_rulename += "-" + indexname + title = "%s (%s)" % (sigmaparser.parsedyaml["title"], index) + else: + title = sigmaparser.parsedyaml["title"] + try: + title = self.options["prefix"] + title + except KeyError: + pass + + self.kibanaconf.append({ + "_id": final_rulename, + "_type": "search", + "_source": { + "title": title, + "description": description, + "hits": 0, + "columns": columns, + "sort": ["@timestamp", "desc"], + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": json.dumps({ + "index": index, + "filter": [], + "highlight": { + "pre_tags": ["@kibana-highlighted-field@"], + "post_tags": ["@/kibana-highlighted-field@"], + "fields": { "*":{} }, + "require_field_match": False, + "fragment_size": 2147483647 + }, + "query": { + "query_string": { + "query": result, + "analyze_wildcard": True + } + } + } + ) + } + } + }) + + def finalize(self): + self.output.print(json.dumps(self.kibanaconf, indent=2)) + +class XPackWatcherBackend(ElasticsearchQuerystringBackend, MultiRuleOutputMixin): + """Converts Sigma Rule into X-Pack Watcher JSON for alerting""" + identifier = "xpack-watcher" + active = True + output_class = SingleOutput + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.watcher_alert = dict() + try: + self.output_type = self.options["output"] + except KeyError: + self.output_type = "curl" + + try: + self.es = self.options["es"] + except KeyError: + self.es = "localhost:9200" + + def generate(self, sigmaparser): + # get the details if this alert occurs + rulename = self.getRuleName(sigmaparser) + description = sigmaparser.parsedyaml.setdefault("description", "") + false_positives = sigmaparser.parsedyaml.setdefault("falsepositives", "") + level = sigmaparser.parsedyaml.setdefault("level", "") + logging_result = "Rule description: "+str(description)+", false positives: "+str(false_positives)+", level: "+level + # Get time frame if exists + interval = sigmaparser.parsedyaml["detection"].setdefault("timeframe", "30m") + + # creating condition + indices = sigmaparser.get_logsource().index + if len(indices) == 0: + indices = ["logstash-*"] + + for condition in sigmaparser.condparsed: + result = self.generateNode(condition.parsedSearch) + try: + if condition.parsedAgg.cond_op == ">": + alert_condition = { "gt": int(condition.parsedAgg.condition) } + elif condition.parsedAgg.cond_op == ">=": + alert_condition = { "gte": int(condition.parsedAgg.condition) } + elif condition.parsedAgg.cond_op == "<": + alert_condition = { "lt": int(condition.parsedAgg.condition) } + elif condition.parsedAgg.cond_op == "<=": + alert_condition = { "lte": int(condition.parsedAgg.condition) } + else: + alert_condition = {"not_eq": 0} + except KeyError: + alert_condition = {"not_eq": 0} + except AttributeError: + alert_condition = {"not_eq": 0} + + self.watcher_alert[rulename] = { + "trigger": { + "schedule": { + "interval": interval # how often the watcher should check + } + }, + "input": { + "search": { + "request": { + "body": { + "size": 0, + "query": { + "query_string": { + "query": result, # this is where the elasticsearch query syntax goes + "analyze_wildcard": True + } + } + }, + "indices": indices + } + } + }, + "condition": { + "compare": { # TODO: Issue #49 + "ctx.payload.hits.total": alert_condition + } + }, + "actions": { + "logging-action": { + "logging": { + "text": logging_result + } + } + } + } + + def finalize(self): + for rulename, rule in self.watcher_alert.items(): + if self.output_type == "plain": # output request line + body + self.output.print("PUT _xpack/watcher/watch/%s\n%s\n" % (rulename, json.dumps(rule, indent=2))) + elif self.output_type == "curl": # output curl command line + self.output.print("curl -s -XPUT --data-binary @- %s/_xpack/watcher/watch/%s <