From cb9aeac7d943caf336b69052dfd36741742409fa Mon Sep 17 00:00:00 2001 From: Thomas Patzke Date: Mon, 23 Oct 2017 00:05:12 +0200 Subject: [PATCH] Added default index handling * Removed default index handling from backend code * Added default indices to config templates --- tools/backends.py | 4 ---- tools/config/elk-linux.yml | 1 + tools/config/elk-windows.yml | 1 + tools/sigma.py | 24 ++++++++++++++++++++---- 4 files changed, 22 insertions(+), 8 deletions(-) diff --git a/tools/backends.py b/tools/backends.py index 96aa31b2b..d883d9c75 100644 --- a/tools/backends.py +++ b/tools/backends.py @@ -329,8 +329,6 @@ class KibanaBackend(ElasticsearchQuerystringBackend, MultiRuleOutputMixin): pass indices = sigmaparser.get_logsource().index - if len(indices) == 0: - indices = ["logstash-*"] for parsed in sigmaparser.condparsed: result = self.generateNode(parsed.parsedSearch) @@ -414,8 +412,6 @@ class XPackWatcherBackend(ElasticsearchQuerystringBackend, MultiRuleOutputMixin) # creating condition indices = sigmaparser.get_logsource().index - if len(indices) == 0: - indices = ["logstash-*"] for condition in sigmaparser.condparsed: result = self.generateNode(condition.parsedSearch) diff --git a/tools/config/elk-linux.yml b/tools/config/elk-linux.yml index 9a32c2706..9b2d48083 100644 --- a/tools/config/elk-linux.yml +++ b/tools/config/elk-linux.yml @@ -12,3 +12,4 @@ logsources: fieldmappings: client_ip: clientip url: request +defaultindex: logstash-* diff --git a/tools/config/elk-windows.yml b/tools/config/elk-windows.yml index 8f77de58a..034539b03 100644 --- a/tools/config/elk-windows.yml +++ b/tools/config/elk-windows.yml @@ -22,3 +22,4 @@ logsources: service: dns-server conditions: EventLog: 'DNS Server' +defaultindex: logstash-* diff --git a/tools/sigma.py b/tools/sigma.py index c10b5665e..16fc202c7 100644 --- a/tools/sigma.py +++ b/tools/sigma.py @@ -711,6 +711,7 @@ class SigmaConfiguration: self.fieldmappings = dict() self.logsources = dict() self.logsourcemerging = SigmaLogsourceConfiguration.MM_AND + self.defaultindex = None self.backend = None else: config = yaml.safe_load(configyaml) @@ -730,6 +731,11 @@ class SigmaConfiguration: except KeyError: self.logsourcemerging = SigmaLogsourceConfiguration.MM_AND + try: + self.defaultindex = config['defaultindex'] + except KeyError: + self.defaultindex = None + self.logsources = list() self.backend = None @@ -743,7 +749,7 @@ class SigmaConfiguration: def get_logsource(self, category, product, service): """Return merged log source definition of all logosurces that match criteria""" matching = [logsource for logsource in self.logsources if logsource.matches(category, product, service)] - return SigmaLogsourceConfiguration(matching) + return SigmaLogsourceConfiguration(matching, self.defaultindex) def set_backend(self, backend): """Set backend. This is used by other code to determine target properties for index addressing""" @@ -754,7 +760,7 @@ class SigmaConfiguration: if type(logsources) != dict: raise SigmaConfigParseError("Logsources must be a map") for name, logsource in logsources.items(): - self.logsources.append(SigmaLogsourceConfiguration(logsource, name, self.logsourcemerging, self.get_indexfield())) + self.logsources.append(SigmaLogsourceConfiguration(logsource, self.defaultindex, name, self.logsourcemerging, self.get_indexfield())) def get_indexfield(self): """Get index condition if index field name is configured""" @@ -766,7 +772,7 @@ class SigmaLogsourceConfiguration: MM_AND = "and" # Merge all conditions with AND MM_OR = "or" # Merge all conditions with OR - def __init__(self, logsource=None, name=None, mergemethod=MM_AND, indexfield=None): + def __init__(self, logsource=None, defaultindex=None, name=None, mergemethod=MM_AND, indexfield=None): self.name = name self.indexfield = indexfield if logsource == None: # create empty object @@ -798,6 +804,13 @@ class SigmaLogsourceConfiguration: # Merge all index patterns self.index = list(set([index for ls in logsource for index in ls.index])) # unique(flat(logsources.index)) + if len(self.index) == 0 and defaultindex is not None: # if no index pattern matched and default index is present: use default index + if type(defaultindex) == str: + self.index = [defaultindex] + elif type(defaultindex) == list and all([type(i) == str for i in defaultindex]): + self.index = defaultindex + else: + raise TypeError("Default index must be string or list of strings") # "merge" index field (should never differ between instances because it is provided by backend class indexfields = [ ls.indexfield for ls in logsource if ls.indexfield != None ] @@ -844,13 +857,16 @@ class SigmaLogsourceConfiguration: index = logsource['index'] if type(index) not in (str, list): raise SigmaConfigParseError("Logsource index must be string or list of strings") - if type(index) == list and not set([type(index) for index in logsource['index']]).issubset({str}): + if type(index) == list and not all([type(index) == str for index in logsource['index']]): raise SigmaConfigParseError("Logsource index patterns must be strings") if type(index) == list: self.index = index else: self.index = [ index ] else: + # no default index handling here - this branch is executed if log source definitions are parsed from + # config and these must not necessarily contain an index definition. A valid index may later be result + # from a merge, where default index handling applies. self.index = [] if 'conditions' in logsource: