From 6bf010fb4b752681c630b9f5110a81e7d643724d Mon Sep 17 00:00:00 2001 From: Florian GAULTIER Date: Mon, 27 May 2019 17:11:59 +0200 Subject: [PATCH 1/4] introduce elastalert-dsl (cherry picked from commit 0235ec23200e62766d9f21fbd26ed834991a0b61) --- tools/config/elk-defaultindex-filebeat.yml | 1 + tools/config/elk-defaultindex-logstash.yml | 1 + tools/config/elk-defaultindex.yml | 1 + tools/config/elk-linux.yml | 1 + tools/config/elk-windows.yml | 3 +- tools/config/elk-winlogbeat.yml | 3 +- tools/config/helk.yml | 1 + tools/sigma/backends/base.py | 2 +- tools/sigma/backends/elasticsearch.py | 44 ++++++++++++++++------ 9 files changed, 42 insertions(+), 15 deletions(-) diff --git a/tools/config/elk-defaultindex-filebeat.yml b/tools/config/elk-defaultindex-filebeat.yml index b38a2c07a..16e1ff0e4 100644 --- a/tools/config/elk-defaultindex-filebeat.yml +++ b/tools/config/elk-defaultindex-filebeat.yml @@ -6,5 +6,6 @@ backends: - kibana - xpack-watcher - elastalert + - elastalert-dsl defaultindex: - filebeat-* diff --git a/tools/config/elk-defaultindex-logstash.yml b/tools/config/elk-defaultindex-logstash.yml index 3fed6bf0e..496139710 100644 --- a/tools/config/elk-defaultindex-logstash.yml +++ b/tools/config/elk-defaultindex-logstash.yml @@ -6,5 +6,6 @@ backends: - kibana - xpack-watcher - elastalert + - elastalert-dsl defaultindex: - logstash-* diff --git a/tools/config/elk-defaultindex.yml b/tools/config/elk-defaultindex.yml index 37f25e353..3870b2359 100644 --- a/tools/config/elk-defaultindex.yml +++ b/tools/config/elk-defaultindex.yml @@ -6,6 +6,7 @@ backends: - kibana - xpack-watcher - elastalert + - elastalert-dsl defaultindex: - logstash-* - filebeat-* diff --git a/tools/config/elk-linux.yml b/tools/config/elk-linux.yml index b82c88172..9eace7fef 100644 --- a/tools/config/elk-linux.yml +++ b/tools/config/elk-linux.yml @@ -6,6 +6,7 @@ backends: - kibana - xpack-watcher - elastalert + - elastalert-dsl logsources: apache: category: webserver diff --git a/tools/config/elk-windows.yml b/tools/config/elk-windows.yml index 88e7486d6..24dd92fa9 100644 --- a/tools/config/elk-windows.yml +++ b/tools/config/elk-windows.yml @@ -6,6 +6,7 @@ backends: - kibana - xpack-watcher - elastalert + - elastalert-dsl logsources: windows: product: windows @@ -38,6 +39,6 @@ logsources: windows-dhcp: product: windows service: dhcp - conditions: + conditions: source: 'Microsoft-Windows-DHCP-Server/Operational' defaultindex: logstash-* diff --git a/tools/config/elk-winlogbeat.yml b/tools/config/elk-winlogbeat.yml index 7bf64c962..cad8f963c 100644 --- a/tools/config/elk-winlogbeat.yml +++ b/tools/config/elk-winlogbeat.yml @@ -6,6 +6,7 @@ backends: - kibana - xpack-watcher - elastalert + - elastalert-dsl logsources: windows: product: windows @@ -38,7 +39,7 @@ logsources: windows-dhcp: product: windows service: dhcp - conditions: + conditions: source: 'Microsoft-Windows-DHCP-Server/Operational' defaultindex: winlogbeat-* # Extract all field names qith yq: diff --git a/tools/config/helk.yml b/tools/config/helk.yml index 14af0ff54..409ecaafb 100644 --- a/tools/config/helk.yml +++ b/tools/config/helk.yml @@ -6,6 +6,7 @@ backends: - kibana - xpack-watcher - elastalert + - elastalert-dsl logsources: windows-application: product: windows diff --git a/tools/sigma/backends/base.py b/tools/sigma/backends/base.py index ffcb7b432..2591f8dfb 100644 --- a/tools/sigma/backends/base.py +++ b/tools/sigma/backends/base.py @@ -25,7 +25,7 @@ from .mixins import RulenameCommentMixin, QuoteCharMixin class BackendOptions(dict): """ Object containing all the options that should be passed to the backend. - + The options can come from command line and a YAML configuration file, and will be merged together. Options from the command line take precedence. """ diff --git a/tools/sigma/backends/elasticsearch.py b/tools/sigma/backends/elasticsearch.py index a723a5767..b0abd67fd 100644 --- a/tools/sigma/backends/elasticsearch.py +++ b/tools/sigma/backends/elasticsearch.py @@ -578,9 +578,8 @@ class XPackWatcherBackend(ElasticsearchQuerystringBackend, MultiRuleOutputMixin) raise NotImplementedError("Output type '%s' not supported" % self.output_type) return result -class ElastalertBackend(MultiRuleOutputMixin, ElasticsearchQuerystringBackend): +class ElastalertBackend(MultiRuleOutputMixin): """Elastalert backend""" - identifier = 'elastalert' active = True supported_alert_methods = {'email', 'http_post'} @@ -636,6 +635,7 @@ class ElastalertBackend(MultiRuleOutputMixin, ElasticsearchQuerystringBackend): "realert": self.generateTimeframe(self.realert_time), #"exponential_realert": self.generateTimeframe(self.expo_realert_time) } + rule_object['filter'] = self.generateQuery(parsed) #Handle aggregation @@ -712,10 +712,6 @@ class ElastalertBackend(MultiRuleOutputMixin, ElasticsearchQuerystringBackend): #Clear fields self.fields = [] - def generateQuery(self, parsed): - #Generate ES QS Query - return [{ 'query' : { 'query_string' : { 'query' : super().generateQuery(parsed) } } }] - def generateNode(self, node): #Save fields for adding them in query_key #if type(node) == sigma.parser.NodeSubexpression: @@ -751,12 +747,12 @@ class ElastalertBackend(MultiRuleOutputMixin, ElasticsearchQuerystringBackend): raise NotImplementedError("%s : The '%s' aggregation operator is not yet implemented for this backend"%(self.title, funcname)) def convertLevel(self, level): - return { - 'critical': 1, - 'high': 2, - 'medium': 3, - 'low': 4 - }.get(level, 2) + return { + 'critical': 1, + 'high': 2, + 'medium': 3, + 'low': 4 + }.get(level, 2) def finalize(self): result = "" @@ -764,3 +760,27 @@ class ElastalertBackend(MultiRuleOutputMixin, ElasticsearchQuerystringBackend): result += yaml.dump(rule, default_flow_style=False) result += '\n' return result + +class ElastalertBackendDsl(ElastalertBackend, ElasticsearchDSLBackend): + """Elastalert backend""" + identifier = 'elastalert-dsl' + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def generateQuery(self, parsed): + #Generate ES DSL Query + super().generateBefore(parsed) + super().generateQuery(parsed) + super().generateAfter(parsed) + return self.queries + +class ElastalertBackendQs(ElastalertBackend, ElasticsearchQuerystringBackend): + """Elastalert backend""" + identifier = 'elastalert' + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def generateQuery(self, parsed): + #Generate ES QS Query + return [{ 'query' : { 'query_string' : { 'query' : super().generateQuery(parsed) } } }] + From 748ac2e20685cf6d3be7cbdce3ca92b89ff7166a Mon Sep 17 00:00:00 2001 From: Florian GAULTIER Date: Wed, 29 May 2019 16:05:53 +0200 Subject: [PATCH 2/4] Dont combine multiple queries --- tools/sigma/backends/elasticsearch.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/sigma/backends/elasticsearch.py b/tools/sigma/backends/elasticsearch.py index b0abd67fd..42739f4fe 100644 --- a/tools/sigma/backends/elasticsearch.py +++ b/tools/sigma/backends/elasticsearch.py @@ -744,7 +744,7 @@ class ElastalertBackend(MultiRuleOutputMixin): if idx == agg.aggfunc: funcname = name break - raise NotImplementedError("%s : The '%s' aggregation operator is not yet implemented for this backend"%(self.title, funcname)) + raise NotImplementedError("%s : The '%s' aggregation operator is not yet implemented for this backend"%(self.title, funcname)) def convertLevel(self, level): return { @@ -772,7 +772,7 @@ class ElastalertBackendDsl(ElastalertBackend, ElasticsearchDSLBackend): super().generateBefore(parsed) super().generateQuery(parsed) super().generateAfter(parsed) - return self.queries + return [self.queries[-1]] class ElastalertBackendQs(ElastalertBackend, ElasticsearchQuerystringBackend): """Elastalert backend""" From 89c1d7b63dd5a1781a78156b57a8b5931c18bf85 Mon Sep 17 00:00:00 2001 From: Florian GAULTIER Date: Wed, 29 May 2019 16:10:14 +0200 Subject: [PATCH 3/4] Wrong fix, self.queries should be emptied after copied to rule_object --- tools/sigma/backends/elasticsearch.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/sigma/backends/elasticsearch.py b/tools/sigma/backends/elasticsearch.py index 42739f4fe..858b413d5 100644 --- a/tools/sigma/backends/elasticsearch.py +++ b/tools/sigma/backends/elasticsearch.py @@ -637,6 +637,7 @@ class ElastalertBackend(MultiRuleOutputMixin): } rule_object['filter'] = self.generateQuery(parsed) + self.queries = [] #Handle aggregation if parsed.parsedAgg: @@ -772,7 +773,7 @@ class ElastalertBackendDsl(ElastalertBackend, ElasticsearchDSLBackend): super().generateBefore(parsed) super().generateQuery(parsed) super().generateAfter(parsed) - return [self.queries[-1]] + return self.queries class ElastalertBackendQs(ElastalertBackend, ElasticsearchQuerystringBackend): """Elastalert backend""" From 67707b6c826582cd1dae0674b2c73cbfe1a96964 Mon Sep 17 00:00:00 2001 From: Thomas Patzke Date: Thu, 30 May 2019 22:38:12 +0200 Subject: [PATCH 4/4] Added test for new elastalert-dsl backend --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index b3517a025..e801dcf93 100644 --- a/Makefile +++ b/Makefile @@ -26,6 +26,7 @@ test-sigmac: coverage run -a --include=$(COVSCOPE) tools/sigmac -rvdI -t graylog rules/ > /dev/null coverage run -a --include=$(COVSCOPE) tools/sigmac -rvdI -t xpack-watcher -c tools/config/elk-winlogbeat.yml rules/ > /dev/null coverage run -a --include=$(COVSCOPE) tools/sigmac -rvdI -t elastalert -c tools/config/elk-winlogbeat.yml -O alert_methods=http_post,email -O emails=test@test.invalid -O http_post_url=http://test.invalid rules/ > /dev/null + coverage run -a --include=$(COVSCOPE) tools/sigmac -rvdI -t elastalert-dsl -c tools/config/elk-winlogbeat.yml -O alert_methods=http_post,email -O emails=test@test.invalid -O http_post_url=http://test.invalid rules/ > /dev/null ! coverage run -a --include=$(COVSCOPE) tools/sigmac -rvdI -t splunk rules/ > /dev/null coverage run -a --include=$(COVSCOPE) tools/sigmac -rvdI -t splunk -c tools/config/splunk-windows-all-index.yml rules/ > /dev/null coverage run -a --include=$(COVSCOPE) tools/sigmac -rvdI -t splunkxml -c tools/config/splunk-windows-all-index.yml rules/ > /dev/null