Merge pull request #737 from NVISO-BE/backend-ee-outliers

ee-outliers backend

Makefile

@@ -30,6 +30,7 @@ test-sigmac:
 	$(COVERAGE) run -a --include=$(COVSCOPE) tools/sigmac -rvdI -t xpack-watcher -O email,index,webhook -c tools/config/winlogbeat.yml rules/ > /dev/null
 	$(COVERAGE) run -a --include=$(COVSCOPE) tools/sigmac -rvdI -t elastalert -c tools/config/winlogbeat.yml -O alert_methods=http_post,email -O emails=test@test.invalid -O http_post_url=http://test.invalid rules/ > /dev/null
 	$(COVERAGE) run -a --include=$(COVSCOPE) tools/sigmac -rvdI -t elastalert-dsl -c tools/config/winlogbeat.yml -O alert_methods=http_post,email -O emails=test@test.invalid -O http_post_url=http://test.invalid rules/ > /dev/null
+	$(COVERAGE) run -a --include=$(COVSCOPE) tools/sigmac -rvdI -t ee-outliers -c tools/config/winlogbeat.yml rules/ > /dev/null
 	! $(COVERAGE) run -a --include=$(COVSCOPE) tools/sigmac -rvdI -t splunk rules/ > /dev/null
 	$(COVERAGE) run -a --include=$(COVSCOPE) tools/sigmac -rvdI -t splunk -c tools/config/splunk-windows-index.yml rules/ > /dev/null
 	$(COVERAGE) run -a --include=$(COVSCOPE) tools/sigmac -rvdI -t splunkxml -c tools/config/splunk-windows.yml rules/ > /dev/null

README.md

@@ -100,7 +100,7 @@ merges multiple YAML documents of a Sigma rule collection into simple Sigma rule
 
 ```
 usage: sigmac [-h] [--recurse] [--filter FILTER]
-              [--target {arcsight,es-qs,es-dsl,kibana,xpack-watcher,elastalert,graylog,limacharlie,logpoint,grep,netwitness,powershell,qradar,qualys,splunk,splunkxml,sumologic,fieldlist,mdatp}]
+              [--target {arcsight,es-qs,es-dsl,kibana,xpack-watcher,elastalert,graylog,limacharlie,logpoint,grep,netwitness,powershell,qradar,qualys,splunk,splunkxml,sumologic,fieldlist,mdatp,ee-outliers}]
               [--target-list] [--config CONFIG] [--output OUTPUT]
               [--backend-option BACKEND_OPTION] [--defer-abort]
               [--ignore-backend-errors] [--verbose] [--debug]
@@ -201,6 +201,7 @@ tools/sigmac -t splunk -c ~/my-splunk-mapping.yml -c tools/config/generic/window
 * [PowerShell](https://docs.microsoft.com/en-us/powershell/scripting/getting-started/getting-started-with-windows-powershell?view=powershell-6)
 * [Grep](https://www.gnu.org/software/grep/manual/grep.html) with Perl-compatible regular expression support
 * [LimaCharlie](https://limacharlie.io)
+* [ee-outliers](https://github.com/NVISO-BE/ee-outliers)
 
 Current work-in-progress
 * [Splunk Data Models](https://docs.splunk.com/Documentation/Splunk/7.1.0/Knowledge/Aboutdatamodels)

tools/config/ecs-proxy.yml

@@ -8,6 +8,7 @@ backends:
   - xpack-watcher
   - elastalert
   - elastalert-dsl
+  - ee-outliers
 logsources:
   proxy:
     category: proxy

tools/config/ecs-zeek-corelight.yml

@@ -13,6 +13,7 @@ backends:
   - corelight_xpack-watcher
   - elastalert
   - elastalert-dsl
+  - ee-outliers
 logsources:
   zeek:
     product: zeek

tools/config/ecs-zeek-elastic-beats-implementation.yml

@@ -8,6 +8,7 @@ backends:
   - xpack-watcher
   - elastalert
   - elastalert-dsl
+  - ee-outliers
 logsources:
   zeek:
     product: zeek

tools/config/filebeat-defaultindex.yml

@@ -8,5 +8,6 @@ backends:
   - xpack-watcher
   - elastalert
   - elastalert-dsl
+  - ee-outliers
 defaultindex:
     - filebeat-*

tools/config/logstash-defaultindex.yml

@@ -8,5 +8,6 @@ backends:
   - xpack-watcher
   - elastalert
   - elastalert-dsl
+  - ee-outliers
 defaultindex:
     - logstash-*

tools/config/logstash-linux.yml

@@ -8,6 +8,7 @@ backends:
   - xpack-watcher
   - elastalert
   - elastalert-dsl
+  - ee-outliers
 logsources:
     apache:
         category: webserver

tools/config/logstash-windows.yml

@@ -8,6 +8,7 @@ backends:
   - xpack-watcher
   - elastalert
   - elastalert-dsl
+  - ee-outliers
 logsources:
   windows:
     product: windows

tools/config/winlogbeat-modules-enabled.yml

@@ -9,6 +9,7 @@ backends:
   - elastalert
   - elastalert-dsl
   - elasticsearch-rule
+  - ee-outliers
 logsources:
   windows:
     product: windows

tools/config/winlogbeat-old.yml

@@ -8,6 +8,7 @@ backends:
   - xpack-watcher
   - elastalert
   - elastalert-dsl
+  - ee-outliers
 logsources:
   windows:
     product: windows

tools/config/winlogbeat.yml

@@ -8,6 +8,7 @@ backends:
   - xpack-watcher
   - elastalert
   - elastalert-dsl
+  - ee-outliers
 logsources:
   windows:
     product: windows

tools/sigma/backends/ee-outliers.py

@@ -0,0 +1,76 @@
+# ee-outliers backend for sigmac
+# NVISO (@NVISO_Labs)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from .elasticsearch import ElasticsearchDSLBackend
+import json
+import logging
+import configparser
+from .mixins import MultiRuleOutputMixin
+from io import StringIO
+
+
+class OutliersBackend(ElasticsearchDSLBackend, MultiRuleOutputMixin):
+    """ee-outliers backend"""
+    identifier = 'ee-outliers'
+    active = True
+
+    def generate(self, sigmaparser):
+        super().generate(sigmaparser)
+
+        self.tags = sigmaparser.parsedyaml.setdefault("tags", "")
+
+        if len(self.queries) == 1:
+            dsl = json.dumps(self.queries[0])
+        else:
+            dsl = json.dumps(self.queries)
+
+        self.queries = []
+
+        use_case_name = self.getRuleName(sigmaparser)
+
+        index = ''
+        if self.indices is not None and len(self.indices) == 1:
+            index = self.indices[0]
+
+        types = ["Sigma hit"]
+        types.extend(self.tags)
+
+        config_data = {
+            "es_dsl_filter": dsl,
+            "es_index": index,
+            "outlier_type": ", ".join(types),
+            "outlier_reason": "Sigma hit - " + self.title,
+            "outlier_summary": "Sigma hit - " + self.title,
+            "run_model": 1,
+            "test_model": 0
+        }
+
+        config = configparser.ConfigParser(interpolation=None)
+        config["simplequery_sigma_" + use_case_name] = config_data
+
+        output = StringIO()
+        config.write(output)
+        result = output.getvalue()
+        output.close()
+
+        return result
+    
+    def finalize(self):
+        """
+        Is called after the last file was processed with generate(). The right place if this backend is not intended to
+        look isolated at each rule, but generates an output which incorporates multiple rules, e.g. dashboards.
+        """
+        pass