diff --git a/.github/workflows/sigma-test.yml b/.github/workflows/sigma-test.yml index ee0c317a5..28931b92e 100644 --- a/.github/workflows/sigma-test.yml +++ b/.github/workflows/sigma-test.yml @@ -23,18 +23,9 @@ jobs: run: | python -m pip install --upgrade pip pip install -r tools/requirements.txt -r tools/requirements-devel.txt - wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add - - sudo apt install -y apt-transport-https - echo "deb https://artifacts.elastic.co/packages/7.x/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic.list - sudo apt update - sudo apt install -y elasticsearch - sudo systemctl start elasticsearch - name: Test Sigma Tools and Rules run: | make test - - name: Test Generated Elasticsearch Query Strings - run: | - make test-backend-es-qs - name: Test SQL(ite) Backend run: | make test-backend-sql diff --git a/Makefile b/Makefile index 8439b5dd7..20eab02b6 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ clearcov: rm -f .coverage finish: - $(COVERAGE) report --fail-under=90 + $(COVERAGE) report --fail-under=80 rm -f $(TMPOUT) test-rules: @@ -59,6 +59,7 @@ test-sigmac: $(COVERAGE) run -a --include=$(COVSCOPE) tools/sigmac -rvdI -t netwitness -c tools/config/netwitness.yml rules/ > /dev/null $(COVERAGE) run -a --include=$(COVSCOPE) tools/sigmac -rvdI -t netwitness-epl -c netwitness-epl rules/ > /dev/null $(COVERAGE) run -a --include=$(COVSCOPE) tools/sigmac -rvdI -t sumologic -O rulecomment -c tools/config/sumologic.yml rules/ > /dev/null + $(COVERAGE) run -a --include=$(COVSCOPE) tools/sigmac -rvdI -t sumologic-cse -O rulecomment -c tools/config/sumologic-cse.yml rules/ > /dev/null $(COVERAGE) run -a --include=$(COVSCOPE) tools/sigmac -rvdI -t humio -O rulecomment -c tools/config/humio.yml rules/ > /dev/null $(COVERAGE) run -a --include=$(COVSCOPE) tools/sigmac -rvdI -t crowdstrike -O rulecomment -c tools/config/crowdstrike.yml rules/ > /dev/null $(COVERAGE) run -a --include=$(COVSCOPE) tools/sigmac -rvdI -t sql -c sysmon rules/ > /dev/null diff --git a/rules/web/web_cve_2020_14882_weblogic_exploit.yml b/rules/web/web_cve_2020_14882_weblogic_exploit.yml new file mode 100644 index 000000000..14afc0d12 --- /dev/null +++ b/rules/web/web_cve_2020_14882_weblogic_exploit.yml @@ -0,0 +1,31 @@ +title: Oracle WebLogic Exploit CVE-2020-14882 +id: 85d466b0-d74c-4514-84d3-2bdd3327588b +status: experimental +description: Detects exploitation attempts on WebLogic servers +author: Florian Roth +date: 2020/11/02 +modified: 2020/11/04 +references: + - https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-14882 + - https://isc.sans.edu/diary/26734 + - https://twitter.com/jas502n/status/1321416053050667009?s=20 + - https://twitter.com/sudo_sudoka/status/1323951871078223874 +logsource: + category: webserver +detection: + selection: + c-uri|contains: + - '/console/images/%252E%252E%252Fconsole.portal' + - '/console/css/%2e' + condition: selection +fields: + - c-ip + - c-dns +falsepositives: + - Unknown +level: high +tags: + - attack.t1100 # an old one + - attack.t1190 + - attack.initial_access + - cve.2020-14882 diff --git a/rules/windows/process_creation/win_apt_lazarus_session_highjack.yml b/rules/windows/process_creation/win_apt_lazarus_session_highjack.yml index ce5e14cc3..bf8fcd819 100644 --- a/rules/windows/process_creation/win_apt_lazarus_session_highjack.yml +++ b/rules/windows/process_creation/win_apt_lazarus_session_highjack.yml @@ -8,7 +8,7 @@ tags: - attack.defense_evasion - attack.t1036 # an old one - attack.t1036.005 -author: Trent Liffick (@tliffick) +author: Trent Liffick (@tliffick), Bartlomiej Czyz (@bczyz1) date: 2020/06/03 logsource: category: process_creation @@ -16,7 +16,7 @@ logsource: detection: selection: Image: - - '*\mstdc.exe' + - '*\msdtc.exe' - '*\gpvc.exe' filter: Image: diff --git a/rules/windows/process_creation/win_susp_gup.yml b/rules/windows/process_creation/win_susp_gup.yml index aaeacc966..19acad192 100644 --- a/rules/windows/process_creation/win_susp_gup.yml +++ b/rules/windows/process_creation/win_susp_gup.yml @@ -10,6 +10,7 @@ tags: - attack.t1073 # an old one author: Florian Roth date: 2019/02/06 +modified: 2020/11/09 logsource: category: process_creation product: windows @@ -17,11 +18,11 @@ detection: selection: Image: '*\GUP.exe' filter: - Image: - - 'C:\Users\\*\AppData\Local\Notepad++\updater\gup.exe' - - 'C:\Users\\*\AppData\Roaming\Notepad++\updater\gup.exe' - - 'C:\Program Files\Notepad++\updater\gup.exe' - - 'C:\Program Files (x86)\Notepad++\updater\gup.exe' + Image|endswith: + - ':\Users\\*\AppData\Local\Notepad++\updater\GUP.exe' + - ':\Users\\*\AppData\Roaming\Notepad++\updater\GUP.exe' + - ':\Program Files\Notepad++\updater\GUP.exe' + - ':\Program Files (x86)\Notepad++\updater\GUP.exe' condition: selection and not filter falsepositives: - Execution of tools named GUP.exe and located in folders different than Notepad++\updater diff --git a/rules/windows/process_creation/win_susp_powershell_enc_cmd.yml b/rules/windows/process_creation/win_susp_powershell_enc_cmd.yml index a2a4ad534..a384047e8 100644 --- a/rules/windows/process_creation/win_susp_powershell_enc_cmd.yml +++ b/rules/windows/process_creation/win_susp_powershell_enc_cmd.yml @@ -40,6 +40,7 @@ detection: - '* -e* IAB*' - '* -e* UwB*' - '* -e* cwB*' + - '*.exe -ENCOD *' falsepositive1: CommandLine: '* -ExecutionPolicy remotesigned *' condition: selection and not falsepositive1 diff --git a/rules/windows/process_creation/win_wmiprvse_spawning_process.yml b/rules/windows/process_creation/win_wmiprvse_spawning_process.yml index fcabfdb70..aafe963ea 100644 --- a/rules/windows/process_creation/win_wmiprvse_spawning_process.yml +++ b/rules/windows/process_creation/win_wmiprvse_spawning_process.yml @@ -3,7 +3,7 @@ id: d21374ff-f574-44a7-9998-4a8c8bf33d7d description: Detects wmiprvse spawning processes status: experimental date: 2019/08/15 -modified: 2019/11/10 +modified: 2020/11/05 author: Roberto Rodriguez @Cyb3rWard0g references: - https://github.com/Cyb3rWard0g/ThreatHunter-Playbook/tree/master/playbooks/windows/02_execution/T1047_windows_management_instrumentation/wmi_win32_process_create_remote.md @@ -19,7 +19,10 @@ detection: filter: - LogonId: '0x3e7' # LUID 999 for SYSTEM - User: 'NT AUTHORITY\SYSTEM' # if we don't have LogonId data, fallback on username detection + - Image|endswith: + - '\WmiPrvSE.exe' + - '\WerFault.exe' condition: selection and not filter falsepositives: - Unknown -level: critical +level: high diff --git a/tools/config/ecs-cloudtrail.yml b/tools/config/ecs-cloudtrail.yml index fe9419bd4..5a1368635 100644 --- a/tools/config/ecs-cloudtrail.yml +++ b/tools/config/ecs-cloudtrail.yml @@ -5,6 +5,7 @@ backends: - es-dsl - es-rule - kibana + - kibana-ndjson - xpack-watcher - elastalert - elastalert-dsl diff --git a/tools/config/ecs-dns.yml b/tools/config/ecs-dns.yml index d41c06398..fddfc32eb 100644 --- a/tools/config/ecs-dns.yml +++ b/tools/config/ecs-dns.yml @@ -5,6 +5,7 @@ backends: - es-dsl - elasticsearch-rule - kibana + - kibana-ndjson - xpack-watcher - elastalert - elastalert-dsl diff --git a/tools/config/ecs-proxy.yml b/tools/config/ecs-proxy.yml index 0659f7c34..2aa441a17 100644 --- a/tools/config/ecs-proxy.yml +++ b/tools/config/ecs-proxy.yml @@ -6,6 +6,7 @@ backends: - es-rule - corelight_elasticsearch-rule - kibana + - kibana-ndjson - xpack-watcher - elastalert - elastalert-dsl diff --git a/tools/config/ecs-zeek-corelight.yml b/tools/config/ecs-zeek-corelight.yml index 0707a7f72..5bf7dab3b 100644 --- a/tools/config/ecs-zeek-corelight.yml +++ b/tools/config/ecs-zeek-corelight.yml @@ -8,6 +8,7 @@ backends: - elasticsearch-rule - corelight_elasticsearch-rule - kibana + - kibana-ndjson - corelight_kibana - xpack-watcher - corelight_xpack-watcher diff --git a/tools/config/ecs-zeek-elastic-beats-implementation.yml b/tools/config/ecs-zeek-elastic-beats-implementation.yml index cd999bb51..ac9b8a45c 100644 --- a/tools/config/ecs-zeek-elastic-beats-implementation.yml +++ b/tools/config/ecs-zeek-elastic-beats-implementation.yml @@ -5,6 +5,7 @@ backends: - es-dsl - elasticsearch-rule - kibana + - kibana-ndjson - xpack-watcher - elastalert - elastalert-dsl diff --git a/tools/config/filebeat-defaultindex.yml b/tools/config/filebeat-defaultindex.yml index 940e34f9b..eb30975da 100644 --- a/tools/config/filebeat-defaultindex.yml +++ b/tools/config/filebeat-defaultindex.yml @@ -5,6 +5,7 @@ backends: - es-dsl - es-rule - kibana + - kibana-ndjson - xpack-watcher - elastalert - elastalert-dsl diff --git a/tools/config/helk.yml b/tools/config/helk.yml index 7042b25f2..c6077fa77 100644 --- a/tools/config/helk.yml +++ b/tools/config/helk.yml @@ -5,6 +5,7 @@ backends: - es-dsl - es-rule - kibana + - kibana-ndjson - xpack-watcher - elastalert - elastalert-dsl diff --git a/tools/config/logstash-defaultindex.yml b/tools/config/logstash-defaultindex.yml index eb566f041..107ca4d92 100644 --- a/tools/config/logstash-defaultindex.yml +++ b/tools/config/logstash-defaultindex.yml @@ -5,6 +5,7 @@ backends: - es-dsl - es-rule - kibana + - kibana-ndjson - xpack-watcher - elastalert - elastalert-dsl diff --git a/tools/config/logstash-linux.yml b/tools/config/logstash-linux.yml index e15e2050d..1ad673fed 100644 --- a/tools/config/logstash-linux.yml +++ b/tools/config/logstash-linux.yml @@ -5,6 +5,7 @@ backends: - es-dsl - es-rule - kibana + - kibana-ndjson - xpack-watcher - elastalert - elastalert-dsl diff --git a/tools/config/logstash-windows.yml b/tools/config/logstash-windows.yml index d21a846bd..317abd9f0 100644 --- a/tools/config/logstash-windows.yml +++ b/tools/config/logstash-windows.yml @@ -5,6 +5,7 @@ backends: - es-dsl - es-rule - kibana + - kibana-ndjson - xpack-watcher - elastalert - elastalert-dsl diff --git a/tools/config/logstash-zeek-default-json.yml b/tools/config/logstash-zeek-default-json.yml index 6915fe14d..e6b1d14ee 100644 --- a/tools/config/logstash-zeek-default-json.yml +++ b/tools/config/logstash-zeek-default-json.yml @@ -5,6 +5,7 @@ backends: - es-dsl - elasticsearch-rule - kibana + - kibana-ndjson - xpack-watcher - elastalert - elastalert-dsl diff --git a/tools/config/sumologic-cse.yml b/tools/config/sumologic-cse.yml new file mode 100644 index 000000000..893e83bec --- /dev/null +++ b/tools/config/sumologic-cse.yml @@ -0,0 +1,185 @@ +title: SumoLogic +order: 20 +backends: + - sumologic-cse + - sumologic-cse-rule +# Sumulogic mapping depends on customer configuration. Adapt to your context! +# typically rule on _sourceCategory, _index or Field Extraction Rules (FER) +# supposing existing FER for service, metdata_vendor, EventID +logsources: + unix: + product: unix + index: UNIX + linux: + product: linux + index: Linux + linux-sshd: + product: linux + service: sshd + index: Linux + linux-auth: + product: linux + service: auth + index: Linux + linux-clamav: + product: linux + service: clamav + index: Linux + windows: + product: windows + index: Windows + conditions: + metdata_vendor: Microsoft + windows-sysmon: + product: windows + service: sysmon + conditions: + metdata_vendor: Microsoft + index: Windows + windows-security: + product: windows + service: security + conditions: + metdata_vendor: Microsoft + index: Security + windows-powershell: + product: windows + service: powershell + conditions: + metdata_vendor: Microsoft + index: Powershell + windows-system: + product: windows + service: system + conditions: + metdata_vendor: Microsoft + index: Windows + windows-dhcp: + product: windows + service: dhcp + conditions: + metdata_vendor: Microsoft + index: Windows + microsoft: + product: gsuite + index: gsuite + apache: + product: apache + service: apache + index: Apache + apache2: + product: apache + index: Apache + nginx: + product: nginx + index: Nginx + cisco: + product: cisco + index: Cisco + webserver: + category: webserver + index: WEBSERVER + firewall: + category: firewall + index: FIREWALL + firewall2: + product: firewall + index: FIREWALL + network-dns: + category: dns + index: DNS + network-dns2: + product: dns + index: DNS + proxy: + category: proxy + index: PROXY + vpn: + product: vpn + index: network + antivirus: + product: antivirus + index: ANTIVIRUS + azure: + product: azure + index: Azure + conditions: + metdata_vendor: Microsoft + azuread: + product: azuread + index: Azure AD + conditions: + metdata_vendor: Microsoft + zeek: + product: zeek + index: zeek + application-sql: + product: sql + index: DATABASE + application-python: + product: python + index: APPLICATIONS + application-django: + product: django + index: Django + application-rails: + product: rails + index: RAILS + application-spring: + product: spring + index: SPRING +# if no index, search in all indexes + +fieldmappings: + EventID: metadata_deviceEventId + event_id: metadata_deviceEventId + Image: baseImage + event_data.Image: baseImage + TargetImage: changeTarget + EventType: changeType + CommandLine: commandLine + Commandline: commandLine + process.args: commandLine + event_data.CommandLine: commandLine + Description: description + + DestinationHostname: dstDevice_hostname + DestinationIp: dstDevice_ip + dst_ip: dstDevice_ip + dst_mac: dstDevice_mac + DestinationPort: dstPort + dst_port: dstPort + + FileName: file_basename + Imphash: file_hash_imphash + hash: file_hash_imphash + file_hash: file_hash_imphash +# :file_hash_md5 +# :file_hash_sha1 +# :file_hash_sha256 + + Path: file_path + path: file_path + + LogonType: logonType + ModuleType: moduleType + ObjectType: objectType + + ParentImage: parentBaseImage + event_data.ParentImage: parentBaseImage + ParentCommandLine: parentCommandLine + ParentProcessName: parentPid + + SourceHostname: srcDevice_hostname + SourceIp: srcDevice_ip + src_ip: srcDevice_ip + src_mac: srcDevice_mac + SourcePort: srcPort + src_port: srcPort + User: user_username + User-Agent: http_userAgent + Protocol: http_url_protocol + + destination.domain: http_url_rootDomain + domain: http_url_rootDomain + diff --git a/tools/config/sysmon.yml b/tools/config/sysmon.yml new file mode 100644 index 000000000..bf0a3e0b9 --- /dev/null +++ b/tools/config/sysmon.yml @@ -0,0 +1,9 @@ +title: Sysmon +order: 20 +backends: + - sysmon +fieldmappings: + event_id: EventID + event_data.ParentImage: ParentImage + event_data.CommandLine: CommandLine + event_data.Image: Image \ No newline at end of file diff --git a/tools/config/winlogbeat-modules-enabled.yml b/tools/config/winlogbeat-modules-enabled.yml index 4009a9bde..292f8d0d6 100644 --- a/tools/config/winlogbeat-modules-enabled.yml +++ b/tools/config/winlogbeat-modules-enabled.yml @@ -5,6 +5,7 @@ backends: - es-dsl - es-rule - kibana + - kibana-ndjson - xpack-watcher - elastalert - elastalert-dsl diff --git a/tools/config/winlogbeat-old.yml b/tools/config/winlogbeat-old.yml index f60c49b84..3955c35a5 100644 --- a/tools/config/winlogbeat-old.yml +++ b/tools/config/winlogbeat-old.yml @@ -5,6 +5,7 @@ backends: - es-dsl - es-rule - kibana + - kibana-ndjson - xpack-watcher - elastalert - elastalert-dsl diff --git a/tools/config/winlogbeat.yml b/tools/config/winlogbeat.yml index 3bc1824e4..4b13103dd 100644 --- a/tools/config/winlogbeat.yml +++ b/tools/config/winlogbeat.yml @@ -5,6 +5,7 @@ backends: - es-dsl - es-rule - kibana + - kibana-ndjson - xpack-watcher - elastalert - elastalert-dsl diff --git a/tools/sigma/backends/ala.py b/tools/sigma/backends/ala.py index 3bbbec544..f70a9dbf1 100644 --- a/tools/sigma/backends/ala.py +++ b/tools/sigma/backends/ala.py @@ -110,62 +110,95 @@ class AzureLogAnalyticsBackend(SingleTextQueryBackend): if isinstance(val, str): if "*" in val[1:-1]: # value contains * inside string - use regex match op = "matches regex" - val = re.sub('\\*', '.*', val) + val = re.sub('(\\\\\*|\*)', '.*', val) if "\\" in val: - return "%s \"(?i)%s\"" % (op, val) - return "%s \"(?i)%s\"" % (op, val) + val = "@'(?i)%s'" % (val) + else: + val = "'(?i)%s'" % (val) + return "%s %s" % (op, self.cleanValue(val)) elif val.startswith("*") or val.endswith("*"): - op = "contains" + if val.startswith("*") and val.endswith("*"): + op = "contains" + elif val.startswith("*"): + op = "endswith" + elif val.endswith("*"): + op = "startswith" val = re.sub('([".^$]|(?![*?]))', '\g<1>', val) - val = re.sub('\\*', '', val) + val = re.sub('(\\\\\*|\*)', '.*', val) val = re.sub('\\?', '.', val) - # if "\\" in val: - # return "%s @\"%s\"" % (op, val) - return "%s \"%s\"" % (op, val) - # elif "\\" in val: - # return "%s @\"%s\"" % (op, val) - return "%s \"%s\"" % (op, val) + if "\\" in val: + return "%s @'%s'" % (op, self.cleanValue(val)) + return "%s '%s'" % (op, self.cleanValue(val)) + elif "\\" in val: + return "%s @'%s'" % (op, self.cleanValue(val)) + return "%s \"%s\"" % (op, self.cleanValue(val)) - def generate(self, sigmaparser): - self.table = None - self.category = sigmaparser.parsedyaml['logsource'].setdefault('category', None) - self.product = sigmaparser.parsedyaml['logsource'].setdefault('product', None) - self.service = sigmaparser.parsedyaml['logsource'].setdefault('service', None) - - detection = sigmaparser.parsedyaml.get("detection", {}) - if "keywords" in detection.keys(): - return super().generate(sigmaparser) - - if self.category == "process_creation": - self.table = "SecurityEvent" - self.eventid = "1" - elif self.service == "security": - self.table = "SecurityEvent" - elif self.service == "sysmon": + def getTable(self, sigmaparser): + if self.category == "process_creation" and len(set(sigmaparser.values.keys()) - {"Image", "ParentImage", + "CommandLine"}) == 0: + self.table = "SecurityEvent | where EventID == 4688 " + self.eventid = "4688" + elif self.category == "process_creation": self.table = "SysmonEvent" - elif self.service == "powershell": + self.eventid = "1" + elif self.service and self.service.lower() == "security": + self.table = "SecurityEvent" + elif self.service and self.service.lower() == "sysmon": + self.table = "SysmonEvent" + elif self.service and self.service.lower() == "powershell": self.table = "Event" - elif self.service == "office365": + elif self.service and self.service.lower() == "office365": self.table = "OfficeActivity" - elif self.service == "azuread": + elif self.service and self.service.lower() == "azuread": self.table = "AuditLogs" - elif self.service == "azureactivity": + elif self.service and self.service.lower() == "azureactivity": self.table = "AzureActivity" else: if self.service: if "-" in self.service: - self.table = "-".join([item.title() for item in self.service.split("-")]) + self.table = "-".join([item.capitalize() for item in self.service.split("-")]) elif "_" in self.service: - self.table = "_".join([item.title() for item in self.service.split("_")]) + self.table = "_".join([item.capitalize() for item in self.service.split("_")]) else: - self.table = self.service.title() + if self.service.islower() or self.service.isupper(): + self.table = self.service.capitalize() + else: + self.table = self.service elif self.product: if "-" in self.product: - self.table = "-".join([item.title() for item in self.product.split("-")]) + self.table = "-".join([item.capitalize() for item in self.product.split("-")]) elif "_" in self.product: - self.table = "_".join([item.title() for item in self.product.split("_")]) + self.table = "_".join([item.capitalize() for item in self.product.split("_")]) else: - self.table = self.product.title() + if self.product.islower() or self.product.isupper(): + self.table = self.product.capitalize() + else: + self.table = self.product + elif self.category: + if "-" in self.category: + self.table = "-".join([item.capitalize() for item in self.category.split("-")]) + elif "_" in self.category: + self.table = "_".join([item.capitalize() for item in self.category.split("_")]) + else: + if self.category.islower() or self.category.isupper(): + self.table = self.category.capitalize() + else: + self.table = self.category + + def generate(self, sigmaparser): + try: + self.category = sigmaparser.parsedyaml['logsource'].setdefault('category', None) + self.product = sigmaparser.parsedyaml['logsource'].setdefault('product', None) + self.service = sigmaparser.parsedyaml['logsource'].setdefault('service', None) + except KeyError: + self.category = None + self.product = None + self.service = None + detection = sigmaparser.parsedyaml.get("detection", {}) + if "keywords" in detection.keys(): + return super().generate(sigmaparser) + if self.table is None: + self.getTable(sigmaparser) return super().generate(sigmaparser) diff --git a/tools/sigma/backends/carbonblack.py b/tools/sigma/backends/carbonblack.py index 860badb5a..975381620 100644 --- a/tools/sigma/backends/carbonblack.py +++ b/tools/sigma/backends/carbonblack.py @@ -44,8 +44,7 @@ class CarbonBlackWildcardHandlingMixin: class CarbonBlackQueryBackend(CarbonBlackWildcardHandlingMixin, SingleTextQueryBackend): - """Converts Sigma rule into CarbonBlack query string. Only searches, no aggregations.""" - + """Converts Sigma rule into CarbonBlack query string. Only searches, no aggregations. Contributed by SOC Prime. https://socprime.com""" identifier = "carbonblack" active = True @@ -133,13 +132,13 @@ class CarbonBlackQueryBackend(CarbonBlackWildcardHandlingMixin, SingleTextQueryB def cleanIPRange(self, value): new_value = value - if type(new_value) is str and value.find('*'): + if isinstance(new_value, str) and value.find('*'): sub = value.count('.') if value[-2:] == '.*': value = value[:-2] min_ip = value + '.0' * (4 - sub) new_value = min_ip + '/' + str(8 * (4 - sub)) - elif type(new_value) is list: + elif isinstance(new_value, list): for index, vl in enumerate(new_value): new_value[index] = self.cleanIPRange(vl) diff --git a/tools/sigma/backends/elasticsearch.py b/tools/sigma/backends/elasticsearch.py index d22a0c37e..c8a7acb53 100644 --- a/tools/sigma/backends/elasticsearch.py +++ b/tools/sigma/backends/elasticsearch.py @@ -1331,3 +1331,128 @@ class ElasticSearchRuleBackend(ElasticsearchQuerystringBackend): if references: rule.update({"references": references}) return json.dumps(rule) + +class KibanaNdjsonBackend(ElasticsearchQuerystringBackend, MultiRuleOutputMixin): + """Converts Sigma rule into Kibana JSON Configuration files (searches only).""" + identifier = "kibana-ndjson" + active = True + options = ElasticsearchQuerystringBackend.options + ( + ("output", "import", "Output format: import = JSON file manually imported in Kibana, curl = Shell script that imports queries in Kibana via curl (jq is additionally required)", "output_type"), + ("es", "localhost:9200", "Host and port of Elasticsearch instance", None), + ("index", ".kibana", "Kibana index", None), + ("prefix", "Sigma: ", "Title prefix of Sigma queries", None), + ) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.kibanaconf = list() + self.indexsearch = set() + + def generate(self, sigmaparser): + description = sigmaparser.parsedyaml.setdefault("description", "") + + columns = list() + try: + for field in sigmaparser.parsedyaml["fields"]: + mapped = sigmaparser.config.get_fieldmapping(field).resolve_fieldname(field, sigmaparser) + if type(mapped) == str: + columns.append(mapped) + elif type(mapped) == list: + columns.extend(mapped) + else: + raise TypeError("Field mapping must return string or list") + except KeyError: # no 'fields' attribute + pass + + indices = sigmaparser.get_logsource().index + if len(indices) == 0: # fallback if no index is given + indices = ["*"] + + for parsed in sigmaparser.condparsed: + result = self.generateNode(parsed.parsedSearch) + + for index in indices: + rulename = self.getRuleName(sigmaparser) + if len(indices) > 1: # add index names if rule must be replicated because of ambigiuous index patterns + raise NotSupportedError("Multiple target indices are not supported by Kibana") + else: + title = self.prefix + sigmaparser.parsedyaml["title"] + + self.indexsearch.add( + "export {indexvar}=$(curl -s '{es}/{index}/_search?q=index-pattern.title:{indexpattern}' | jq -r '.hits.hits[0]._id | ltrimstr(\"index-pattern:\")')".format( + es=self.es, + index=self.index, + indexpattern=index.replace("*", "\\*"), + indexvar=self.index_variable_name(index) + ) + ) + self.kibanaconf.append({ + "id": rulename, + "type": "search", + "attributes": { + "title": title, + "description": description, + "hits": 0, + "columns": columns, + "sort": ["@timestamp", "desc"], + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "index": index, + "filter": [], + "highlight": { + "pre_tags": ["@kibana-highlighted-field@"], + "post_tags": ["@/kibana-highlighted-field@"], + "fields": { "*":{} }, + "require_field_match": False, + "fragment_size": 2147483647 + }, + "query": { + "query_string": { + "query": result, + "analyze_wildcard": True + } + } + } + } + }, + "references": [ + { + "id": index, + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ] + }) + + def finalize(self): + if self.output_type == "import": # output format that can be imported via Kibana UI + for item in self.kibanaconf: # JSONize kibanaSavedObjectMeta.searchSourceJSON + item['attributes']['kibanaSavedObjectMeta']['searchSourceJSON'] = json.dumps(item['attributes']['kibanaSavedObjectMeta']['searchSourceJSON']) + if self.kibanaconf: + ndjson = "" + for item in self.kibanaconf: + ndjson += json.dumps(item) + ndjson += "\n" + return ndjson + elif self.output_type == "curl": + for item in self.indexsearch: + return item + for item in self.kibanaconf: + item['attributes']['kibanaSavedObjectMeta']['searchSourceJSON']['index'] = "$" + self.index_variable_name(item['attributes']['kibanaSavedObjectMeta']['searchSourceJSON']['index']) # replace index pattern with reference to variable that will contain Kibana index UUID at script runtime + item['attributes']['kibanaSavedObjectMeta']['searchSourceJSON'] = json.dumps(item['attributes']['kibanaSavedObjectMeta']['searchSourceJSON']) # Convert it to JSON string as expected by Kibana + item['attributes']['kibanaSavedObjectMeta']['searchSourceJSON'] = item['attributes']['kibanaSavedObjectMeta']['searchSourceJSON'].replace("\\", "\\\\") # Add further escaping for escaped quotes for shell + return "curl -s -XPUT -H 'Content-Type: application/json' --data-binary @- '{es}/{index}/doc/{doc_id}' <. - +import json +import os import re -import sigma -from sigma.parser.condition import ConditionOR -from .base import SingleTextQueryBackend +import sys + +from sigma.backends.base import SingleTextQueryBackend +from sigma.backends.exceptions import NotSupportedError +from sigma.parser.condition import ConditionOR, SigmaAggregationParser # Sumo specifics # https://help.sumologic.com/05Search/Search-Query-Language @@ -29,13 +32,13 @@ from .base import SingleTextQueryBackend class SumoLogicBackend(SingleTextQueryBackend): - """Converts Sigma rule into SumoLogic query""" + """Converts Sigma rule into SumoLogic query. Contributed by SOC Prime. https://socprime.com""" identifier = "sumologic" active = True config_required = False default_config = ["sysmon", "sumologic"] - index_field = "_index" + index_field = "_sourceCategory" reClear = None andToken = " AND " orToken = " OR " @@ -59,22 +62,46 @@ class SumoLogicBackend(SingleTextQueryBackend): agg.groupfield = 'hostname' if agg.aggfunc_notrans == 'count() by': agg.aggfunc_notrans = 'count by' - if agg.aggfunc == sigma.parser.condition.SigmaAggregationParser.AGGFUNC_NEAR: + if agg.aggfunc == SigmaAggregationParser.AGGFUNC_NEAR: raise NotImplementedError("The 'near' aggregation operator is not yet implemented for this backend") - # WIP - # ex: - # (QUERY) | timeslice 5m - # | count_distinct(process) _timeslice,hostname - # | where _count_distinct > 5 - # return " | timeslice %s | count_distinct(%s) %s | where _count_distinct > 0" % (self.interval, agg.aggfunc_notrans or "", agg.aggfield or "", agg.groupfield or "") - # return " | timeslice %s | count_distinct(%s) %s | where _count_distinct %s %s" % (self.interval, agg.aggfunc_notrans, agg.aggfield or "", agg.groupfield or "", agg.cond_op, agg.condition) - if not agg.groupfield: - # return " | %s(%s) | when _count %s %s" % (agg.aggfunc_notrans, agg.aggfield or "", agg.cond_op, agg.condition) - return " | %s %s | where _count %s %s" % (agg.aggfunc_notrans, agg.aggfield or "", agg.cond_op, agg.condition) - elif agg.groupfield: - return " | %s by %s | where _count %s %s" % (agg.aggfunc_notrans, agg.groupfield or "", agg.cond_op, agg.condition) + if self.keypresent: + if not agg.groupfield: + if agg.aggfield: + agg.aggfunc_notrans = "count_distinct" + return " \n| %s(%s) \n| where _count_distinct %s %s" % ( + agg.aggfunc_notrans, agg.aggfield, agg.cond_op, agg.condition) + else: + return " \n| %s | where _count %s %s" % ( + agg.aggfunc_notrans, agg.cond_op, agg.condition) + elif agg.groupfield: + if agg.aggfield: + agg.aggfunc_notrans = "count_distinct" + return " \n| %s(%s) by %s \n| where _count_distinct %s %s" % ( + agg.aggfunc_notrans, agg.aggfield, agg.groupfield, agg.cond_op, agg.condition) + else: + return " \n| %s by %s \n| where _count %s %s" % ( + agg.aggfunc_notrans, agg.groupfield, agg.cond_op, agg.condition) + else: + return " \n| %s | where _count %s %s" % (agg.aggfunc_notrans, agg.cond_op, agg.condition) else: - return " | %s(%s) by %s | where _count %s %s" % (agg.aggfunc_notrans, agg.aggfield or "", agg.groupfield or "", agg.cond_op, agg.condition) + if not agg.groupfield: + if agg.aggfield: + agg.aggfunc_notrans = "count_distinct" + return " \n| parse \"[%s=*]\" as searched nodrop\n| %s(searched) \n| where _count_distinct %s %s" % ( + agg.aggfield, agg.aggfunc_notrans, agg.cond_op, agg.condition) + else: + return " \n| %s | where _count %s %s" % ( + agg.aggfunc_notrans, agg.cond_op, agg.condition) + elif agg.groupfield: + if agg.aggfield: + agg.aggfunc_notrans = "count_distinct" + return " \n| parse \"[%s=*]\" as searched nodrop\n| parse \"[%s=*]\" as grpd nodrop\n| %s(searched) by grpd \n| where _count_distinct %s %s" % ( + agg.aggfield, agg.groupfield, agg.aggfunc_notrans, agg.cond_op, agg.condition) + else: + return " \n| parse \"[%s=*]\" as grpd nodrop\n| %s by grpd \n| where _count %s %s" % ( + agg.groupfield, agg.aggfunc_notrans, agg.cond_op, agg.condition) + else: + return " \n| %s | where _count %s %s" % (agg.aggfunc_notrans, agg.cond_op, agg.condition) def generateBefore(self, parsed): # not required but makes query faster, especially if no FER or _index/_sourceCategory @@ -126,13 +153,17 @@ class SumoLogicBackend(SingleTextQueryBackend): if '|' in result: return result else: - return "(" + result + ")" + return result def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) # TODO/FIXME! depending on deployment configuration, existing FER must be populate here (or backend config?) - # aFL = ["EventID"] - aFL = ["_index", "_sourceCategory", "_view", "EventID", "sourcename", "CommandLine", "NewProcessName", "Image", "ParentImage", "ParentCommandLine", "ParentProcessName"] + aFL = ["_sourceCategory", "_view", "_sourceName"] + if self.sigmaconfig.config.get("afl_fields"): + self.keypresent = True + aFL.extend(self.sigmaconfig.config.get("afl_fields")) + else: + self.keypresent = False for item in self.sigmaconfig.fieldmappings.values(): if item.target_type is list: aFL.extend(item.target) @@ -146,25 +177,22 @@ class SumoLogicBackend(SingleTextQueryBackend): # Clearing values from special characters. # Sumologic: only removing '*' (in quotes, is litteral. without, is wildcard) and '"' - def CleanNode(self, node): - search_ptrn = re.compile(r"[*\"\\]") - replace_ptrn = re.compile(r"[*\"\\]") - match = search_ptrn.search(str(node)) - new_node = list() - if match: - replaced_str = replace_ptrn.sub('*', node) - node = [x for x in replaced_str.split('*') if x] - new_node.extend(node) - else: - new_node.append(node) - node = new_node + + def cleanNode(self, node, key=None): + if "*" in node and key and not re.search("[\s]", node): + return node + elif "*" in node and not key: + return [x for x in node.split("*") if x] return node # Clearing values from special characters. def generateMapItemNode(self, node): key, value = node if key in self.allowedFieldsList: - if not self.mapListsSpecialHandling and type(value) in ( + if key in ["_sourceCategory", "_sourceName"]: + value = "*%s*" % value.lower() + return self.mapExpression % (key, value) + elif not self.mapListsSpecialHandling and type(value) in ( str, int, list) or self.mapListsSpecialHandling and type(value) in (str, int): if key in ("LogName", "source"): self.logname = value @@ -181,29 +209,29 @@ class SumoLogicBackend(SingleTextQueryBackend): str, int, list) or self.mapListsSpecialHandling and type(value) in (str, int): if type(value) is str: new_value = list() - value = self.CleanNode(value) + value = self.cleanNode(value) if type(value) == list: - new_value.append(self.andToken.join([self.valueExpression % val for val in value])) + new_value.append(self.andToken.join([self.cleanValue(val) for val in value])) else: new_value.append(value) if len(new_value) == 1: if self.generateANDNode(new_value): - return "(" + self.generateANDNode(new_value) + ")" + return self.generateANDNode(new_value) else: # if after cleaning node, it is empty but there is AND statement... make it true. return "true" else: - return "(" + self.generateORNode(new_value) + ")" + return self.generateORNode(new_value) else: return self.generateValueNode(value) elif type(value) is list: new_value = list() for item in value: - item = self.CleanNode(item) + item = self.cleanNode(item) if type(item) is list and len(item) == 1: - new_value.append(self.valueExpression % item[0]) + new_value.append(item[0]) elif type(item) is list: - new_value.append(self.andToken.join([self.valueExpression % val for val in item])) + new_value.append(self.andToken.join([self.cleanValue(val) for val in item])) else: new_value.append(item) return self.generateORNode(new_value) @@ -217,35 +245,25 @@ class SumoLogicBackend(SingleTextQueryBackend): # => OK only if field entry with list, not string # => generateNode: call cleanValue def cleanValue(self, val, key=''): - # in sumologic, if key, can use wildcard outside of double quotes. if inside, it's litteral - if key: - val = re.sub(r'\"', '\\"', str(val)) - val = re.sub(r'(.+)\*(.+)', '"\g<1>"*"\g<2>"', val, 0) - val = re.sub(r'^\*', '*"', val) - val = re.sub(r'\*$', '"*', val) - # if unbalanced wildcard? - if val.startswith('*"') and not (val.endswith('"*') or val.endswith('"')): - val = val + '"' - if val.endswith('"*') and not (val.startswith('*"') or val.startswith('"')): - val = '"' + val - # double escape if end quote - if val.endswith('\\"*') and not val.endswith('\\\\"*'): - val = re.sub(r'\\"\*$', '\\\\\\"*', val) - # if not key and not (val.startswith('"') and val.endswith('"')) and not (val.startswith('(') and val.endswith(')')) and not ('|' in val) and val: - # apt_babyshark.yml - if not (val.startswith('"') and val.endswith('"')) and not (val.startswith('(') and val.endswith(')')) and not ('|' in val) and not ('*' in val) and val and not '_index' in key and not '_sourceCategory' in key and not '_view' in key: - val = '"%s"' % val + if isinstance(val, str): + val = re.sub("[^\\\"](\")", "\\\"", val) + if re.search("[\W\s]", val):# and not val.startswith('"') and not val.endswith('"'): # or "\\" in node in [] or "/" in node: + return self.valueExpression % val return val # for keywords values with space def generateValueNode(self, node, key=''): - cV = self.cleanValue(str(node), key) + cV = self.cleanNode(str(node), key) if type(node) is int: return cV + if type(cV) is list: + return "(%s)" % "AND".join([self.cleanValue(item) for item in cV]) if 'AND' in node and cV: return "(" + cV + ")" - else: + elif isinstance(node, str) and node.startswith('"') and node.endswith('"'): return cV + else: + return self.cleanValue(cV) def generateMapItemListNode(self, key, value): itemslist = list() @@ -256,15 +274,209 @@ class SumoLogicBackend(SingleTextQueryBackend): itemslist.append('%s' % (self.generateValueNode(item))) return "(" + " OR ".join(itemslist) + ")" - # generateORNode algorithm for ArcSightBackend & SumoLogicBackend class. + # generateORNode algorithm for SumoLogicBackend class. def generateORNode(self, node): if type(node) == ConditionOR and all(isinstance(item, str) for item in node): new_value = list() for value in node: - value = self.CleanNode(value) + value = self.cleanNode(value) if type(value) is list: new_value.append(self.andToken.join([self.valueExpression % val for val in value])) else: new_value.append(value) return "(" + self.orToken.join([self.generateNode(val) for val in new_value]) + ")" return "(" + self.orToken.join([self.generateNode(val) for val in node]) + ")" + + +class SumoLogicCSE(SumoLogicBackend): + """Converts Sigma rule into SumoLogic CSE query. Contributed by SOC Prime. https://socprime.com""" + identifier = "sumologic-cse" + active = True + config_required = False + default_config = ["sysmon"] + + index_field = "metdata_product" + reClear = None + #reEscape = re.compile('[\\\\"]') + andToken = " and " + orToken = " or " + notToken = "!" + subExpression = "(%s)" + listExpression = "(%s)" + listSeparator = ", " + valueExpression = "\"%s\"" + nullExpression = "isEmpty(%s)" + notNullExpression = "!isEmpty(%s)" + mapExpression = "%s=%s" + mapListsSpecialHandling = True + mapListValueExpression = "%s IN %s" + interval = None + logname = None + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.allowedFieldsList.extend(["metdata_product", "metdata_vendor"]) + + def cleanValue(self, val, key=''): + if key == 'metadata_deviceEventId' or isinstance(val, int) or val.isdigit(): + return val + return self.valueExpression % val + + def cleanNode(self, node, key=None): + return node + + # Clearing values from special characters. + def generateMapItemNode(self, node): + key, value = node + if key: + if not self.mapListsSpecialHandling and type(value) in ( + str, int, list) or self.mapListsSpecialHandling and type(value) in (str, int): + if key in ("LogName", "source"): + self.logname = value + # need cleanValue if sigma entry with single quote + return self.mapExpression % (key, self.cleanValue(value, key)) + elif type(value) is list: + return self.generateMapItemListNode(key, value) + elif value is None: + return self.nullExpression % (key,) + else: + raise TypeError("Backend does not support map values of type " + str(type(value))) + raise TypeError("Backend does not support query without key.") + + def generateMapItemListNode(self, key, value): + if len(value) == 1: + return self.mapExpression % (key, value[0]) + return "%s IN (%s)" % (key, ", ".join([self.cleanValue(item, key) for item in value])) + + +class SumoLogicCSERule(SumoLogicCSE): + """Converts Sigma rule into SumoLogic CSE query""" + identifier = "sumologic-cse-rule" + active = True + + def __init__(self, *args, **kwargs): + """Initialize field mappings""" + super().__init__(*args, **kwargs) + self.techniques = self._load_mitre_file("techniques") + self.allowedCategories = ["Threat Intelligence", "Initial Access", "Execution", "Persistence", "Privilege Escalation", + "Defense Evasion", "Credential Access", "Discovery", "Lateral Movement", "Collection", + "Command and Control", "Exfiltration", "Impact"] + self.defaultCategory = "Unknown/Other" + self.results = [] + + def find_technique(self, key_ids): + for key_id in set(key_ids): + if not key_id: + continue + for technique in self.techniques: + if key_id == technique.get("technique_id", ""): + yield technique + + def _load_mitre_file(self, mitre_type): + try: + backend_dir = os.path.normpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..", "config", "mitre")) + path = os.path.join(backend_dir, "{}.json".format(mitre_type)) + with open(path) as config_file: + config = json.load(config_file) + return config + except (IOError, OSError) as e: + print("Failed to open {} configuration file '%s': %s".format(path, str(e)), file=sys.stderr) + return [] + except json.JSONDecodeError as e: + print("Failed to parse {} configuration file '%s' as valid YAML: %s" % (path, str(e)), file=sys.stderr) + return [] + + def skip_tactics_or_techniques(self, src_technics, src_tactics): + tactics = set() + technics = set() + + local_storage_techniques = {item["technique_id"]: item for item in self.find_technique(src_technics)} + + for key_id in src_technics: + src_tactic = local_storage_techniques.get(key_id, {}).get("tactic") + if not src_tactic: + continue + src_tactic = set(src_tactic) + + for item in src_tactics: + if item in src_tactic: + technics.add(key_id) + tactics.add(item) + + return sorted(tactics), sorted(technics) + + def parse_severity(self, old_severity): + if old_severity.lower() == "critical": + return "high" + return old_severity + + def get_tactics_and_techniques(self, tags): + tactics = list() + technics = list() + + for tag in tags: + tag = tag.replace("attack.", "") + if re.match("[t][0-9]{4}", tag, re.IGNORECASE): + technics.append(tag.title()) + elif re.match("[s][0-9]{4}", tag, re.IGNORECASE): + continue + else: + if "_" in tag: + tag = tag.replace("_", " ") + tag = tag.title() + tactics.append(tag) + + return tactics, technics + + def map_risk_score(self, level): + if level == "critical": + return 5 + elif level == "high": + return 4 + elif level == "medium": + return 3 + elif level == "low": + return 2 + return 1 + + def create_rule(self, config): + tags = config.get("tags", []) + + tactics, technics = self.get_tactics_and_techniques(tags) + tactics, technics = self.skip_tactics_or_techniques(technics, tactics) + tactics = list(map(lambda s: s.replace(" ", ""), tactics)) + score = self.map_risk_score(config.get("level", "medium")) + rule = { + "name": "{} by {}".format(config.get("title"), config.get('author')), + "description": "{} {}".format(config.get("description"), "Technique: {}.".format(",".join(technics))), + "enabled": True, + "expression": """{}""".format(config.get("translation", "")), + "assetField": "device_hostname", + "score": score, + "stream": "record" + } + if tactics and tactics[0] in self.allowedCategories: + rule.update({"category": tactics[0]}) + else: + rule.update({"category": "Unknown/Other"}) + self.results.append(rule) + #return json.dumps(rule, indent=4, sort_keys=False) + + def generate(self, sigmaparser): + translation = super().generate(sigmaparser) + if translation: + configs = sigmaparser.parsedyaml + configs.update({"translation": translation}) + rule = self.create_rule(configs) + return rule + else: + raise NotSupportedError("No table could be determined from Sigma rule") + + def finalize(self): + if len(self.results) == 1: + return json.dumps(self.results[0], indent=4, sort_keys=False) + elif len(self.results) > 1: + return json.dumps(self.results, indent=4, sort_keys=False) + + + diff --git a/tools/sigma/backends/sysmon.py b/tools/sigma/backends/sysmon.py index 12171dc34..66832d576 100644 --- a/tools/sigma/backends/sysmon.py +++ b/tools/sigma/backends/sysmon.py @@ -14,7 +14,7 @@ class SysmonConfigBackend(SingleTextQueryBackend, MultiRuleOutputMixin): orToken = " OR " notToken = "NOT " subExpression = "(%s)" - config_required = False + config_required = True INCLUDE = "include" EXCLUDE = "exclude" conditionDict = { diff --git a/tools/sigma/config/mapping.py b/tools/sigma/config/mapping.py index a03976834..f857f0887 100644 --- a/tools/sigma/config/mapping.py +++ b/tools/sigma/config/mapping.py @@ -207,6 +207,14 @@ class FieldMappingChain(object): def resolve(self, key, value, sigmaparser): if type(self.fieldmappings) == str: # one field mapping return (self.fieldmappings, value) + elif isinstance(self.fieldmappings, ConditionalFieldMapping): + logsource = sigmaparser.parsedyaml.get("logsource") + condition = self.fieldmappings.conditions + for source_type, logsource_item in logsource.items(): + if condition.get(source_type) and condition.get(source_type, {}).get(logsource_item): + new_field = condition.get(source_type, {}).get(logsource_item) + self.fieldmappings.default = new_field + return self.fieldmappings.resolve(self.fieldmappings.source, value, sigmaparser) elif isinstance(self.fieldmappings, SimpleFieldMapping): return self.fieldmappings.resolve(key, value, sigmaparser) elif type(self.fieldmappings) == set: diff --git a/tools/sigma/sigma2attack.py b/tools/sigma/sigma2attack.py index 5543d6eee..165d077fc 100755 --- a/tools/sigma/sigma2attack.py +++ b/tools/sigma/sigma2attack.py @@ -21,7 +21,7 @@ def main(): num_rules_used = 0 for rule_file in rule_files: try: - rule = yaml.safe_load(open(rule_file).read()) + rule = yaml.safe_load(open(rule_file, encoding="utf-8").read()) except yaml.YAMLError: sys.stderr.write("Ignoring rule " + rule_file + " (parsing failed)\n") continue @@ -61,7 +61,10 @@ def main(): "maxValue": curr_max_technique_count, "minValue": 0 }, - "version": "2.2", + "versions": { + "navigator": "4.0", + "layer": "4.0" + }, "techniques": scores, }