diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 000000000..1806298b1 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,97 @@ +# Release Notes + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) +from version 0.14.0. + +## Unreleased + +Changes from this section will be contained in the next release. + +## 0.14 + +### Added + +* sigma-similarity tool +* LimaCharlie backend +* Default configurations for some backends that are used if no configuration is passed. +* Regular expression support for es-dsl backend (propagates to backends derived from this like elastalert-dsl) +* Value modifiers: + * startswith + * endswith + +### Changed + +* Removal of line breaks in elastalert output +* Searches not bound to fields are restricted to keyword fields in es-qs backend +* Graylog backend now based on es-qs backend + +### Fixed + +* Removed ProcessCommandLine mapping for Windows Security EventID 4688 in generic + process creation log source configuration. + +## 0.13 + +### Added + +* Index mappings for Sumologic +* Malicious cmdlets in wdatp +* QRadar support for keyword searches +* QRadar mapping improvements +* QRadar field selection +* QRadar type regex modifier support +* Elasticsearch keyword field blacklisting with wildcards +* Added dateField configuration parameter in xpack-watcher backend +* Field mappings in configurations +* Field name mapping for conditional fields +* Value modifiers: + * utf16 + * utf16le + * wide + * utf16be + +### Changed + +* Improved --backend-config help text + +### Fixed + +* Backend errors in ala +* Slash escaping within es-dsl wildcard queries +* QRadar backend config +* QRadar field name and value escaping and handling +* Elasticsearch wildcard detection pattern +* Aggregation on keyword field in es-dsl backend + +## 0.12.1 + +### Fixed + +* Missing build dependency + +## 0.12 + +### Added + +* Usage of "Channel" field in ELK Windows configuration +* Fields to mappings +* xpack-watcher actions index and webhook +* Config for Winlogbeat 7.x +* Value modifiers +* Regular expression support + +### Changed + +* Warning/error messages +* Sumologic value cleaning +* Explicit OR for Elasticsearch query strings +* Listing of available configurations on missing configuration error + +### Fixed + +* Conditions in es-dsl backend +* Sumologic handling of null values +* Ignore timeframe detection keyword in all/any of conditions diff --git a/CHANGELOG.md.j2 b/CHANGELOG.md.j2 new file mode 100644 index 000000000..8dd07eee2 --- /dev/null +++ b/CHANGELOG.md.j2 @@ -0,0 +1,38 @@ +## {{ version.minor }}.{{ version.major }}.{{ version.patch }} ({{ date }}) + +### Added + +{% for item in added %} +* {{ item | indent(2) }} +{% endfor %} + +### Changed + +{% for item in changed %} +* {{ item | indent(2) }} +{% endfor %} + +### Deprecated + +{% for item in deprecated %} +* {{ item | indent(2) }} +{% endfor %} + +### Removed + +{% for item in removed %} +* {{ item | indent(2) }} +{% endfor %} + +### Fixed + +{% for item in fixed %} +* {{ item | indent(2) }} +{% endfor %} + +### Security + +{% for item in security %} +* {{ item | indent(2) }} +{% endfor %} + diff --git a/Pipfile.lock b/Pipfile.lock index e6143397c..776e1a075 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -207,25 +207,25 @@ }, "pymisp": { "hashes": [ - "sha256:1983808d9a834c26d42d52871af1f86dc9739c9f2ee22091cf4a2a62ce6a171d", - "sha256:32675ce303f9d06698eb390c5381cb1de430d355e203612264bce6cd53972b95", - "sha256:9cf1187b5d618bd2b0e631cc877586b7cd5d02b59322a509a4f5ad07496cd171" + "sha256:17b145dbc39a1ba4ebce60e8b75a479d2c8fd3c2a239f32682f2e1a3636469ec", + "sha256:814023f346f9e1dcf6763d93450df44ff0157f2061c612a7eaf2020280f588a3", + "sha256:de67196f6a8916b9c52a84a1c45ea967c53fa9d2b3795b070ad2c1cbc28d79d7" ], "index": "pypi", - "version": "==2.4.117" + "version": "==2.4.117.2" }, "pyrsistent": { "hashes": [ - "sha256:34b47fa169d6006b32e99d4b3c4031f155e6e68ebcc107d6454852e8e0ee6533" + "sha256:eb6545dbeb1aa69ab1fb4809bfbf5a8705e44d92ef8fc7c2361682a47c46c778" ], - "version": "==0.15.4" + "version": "==0.15.5" }, "python-dateutil": { "hashes": [ - "sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb", - "sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e" + "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c", + "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a" ], - "version": "==2.8.0" + "version": "==2.8.1" }, "python-utils": { "hashes": [ @@ -262,19 +262,19 @@ }, "six": { "hashes": [ - "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", - "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" + "sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd", + "sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66" ], - "version": "==1.12.0" + "version": "==1.13.0" }, "typing-extensions": { "hashes": [ - "sha256:2ed632b30bb54fc3941c382decfd0ee4148f5c591651c9272473fea2c6397d95", - "sha256:b1edbbf0652660e32ae780ac9433f4231e7339c7f9a8057d0f042fcbcea49b87", - "sha256:d8179012ec2c620d3791ca6fe2bf7979d979acdbef1fca0bc56b37411db682ed" + "sha256:091ecc894d5e908ac75209f10d5b4f118fbdb2eb1ede6a63544054bb1edb41f2", + "sha256:910f4656f54de5993ad9304959ce9bb903f90aadc7c67a0bef07e678014e892d", + "sha256:cf8b63fedea4d89bab840ecbb93e75578af28f76f66c35889bd7065f5af88575" ], "markers": "python_version < '3.7'", - "version": "==3.7.4" + "version": "==3.7.4.1" }, "urllib3": { "hashes": [ diff --git a/rules/windows/builtin/win_susp_add_sid_history.yml b/rules/windows/builtin/win_susp_add_sid_history.yml index 5f5daa7a0..3452d6138 100644 --- a/rules/windows/builtin/win_susp_add_sid_history.yml +++ b/rules/windows/builtin/win_susp_add_sid_history.yml @@ -25,4 +25,4 @@ detection: condition: selection1 or (selection2 and not selection3) falsepositives: - Migration of an account into a new domain -level: medium +level: low diff --git a/rules/windows/other/win_wmi_persistence.yml b/rules/windows/other/win_wmi_persistence.yml index f978565a5..077f0aacb 100644 --- a/rules/windows/other/win_wmi_persistence.yml +++ b/rules/windows/other/win_wmi_persistence.yml @@ -26,5 +26,5 @@ detection: condition: selection and 1 of keywords or selection2 falsepositives: - Unknown (data set is too small; further testing needed) -level: high +level: medium diff --git a/rules/windows/process_creation/win_shell_spawn_susp_program.yml b/rules/windows/process_creation/win_shell_spawn_susp_program.yml index 0de77853c..66f89ad0a 100644 --- a/rules/windows/process_creation/win_shell_spawn_susp_program.yml +++ b/rules/windows/process_creation/win_shell_spawn_susp_program.yml @@ -18,7 +18,7 @@ detection: ParentImage: - '*\mshta.exe' - '*\powershell.exe' - - '*\cmd.exe' + # - '*\cmd.exe' # too many false positives - '*\rundll32.exe' - '*\cscript.exe' - '*\wscript.exe' diff --git a/rules/windows/process_creation/win_susp_userinit_child.yml b/rules/windows/process_creation/win_susp_userinit_child.yml index bed1fbbfd..5a255ad87 100644 --- a/rules/windows/process_creation/win_susp_userinit_child.yml +++ b/rules/windows/process_creation/win_susp_userinit_child.yml @@ -1,6 +1,6 @@ title: Suspicious Userinit Child Process status: experimental -description: Detects the creation of a process from Windows task manager +description: Detects a suspicious child process of userinit references: - https://twitter.com/SBousseaden/status/1139811587760562176 author: Florian Roth (rule), Samir Bousseaden (idea) @@ -11,14 +11,14 @@ logsource: detection: selection: ParentImage: '*\userinit.exe' - filter: - CommandLine: - - '*\explorer.exe*' - - '*\\netlogon\\*' - condition: selection and not filter + filter1: + CommandLine: '*\\netlogon\\*' + filter2: + Image: '*\explorer.exe' + condition: selection and not filter1 and not filter2 fields: - CommandLine - ParentCommandLine falsepositives: - Administrative scripts -level: high +level: medium diff --git a/rules/windows/sysmon/sysmon_logon_scripts_userinitmprlogonscript.yml b/rules/windows/sysmon/sysmon_logon_scripts_userinitmprlogonscript.yml index 2453d5214..64e171ad8 100644 --- a/rules/windows/sysmon/sysmon_logon_scripts_userinitmprlogonscript.yml +++ b/rules/windows/sysmon/sysmon_logon_scripts_userinitmprlogonscript.yml @@ -15,8 +15,9 @@ detection: exec_selection: EventID: 1 # Migration to process_creation requires multipart YAML ParentImage: '*\userinit.exe' - exec_exclusion: + exec_exclusion1: Image: '*\explorer.exe' + exec_exclusion2: CommandLine: '*\netlogon.bat' create_selection_cli: EventID: @@ -33,7 +34,7 @@ detection: create_keywords_cli: CommandLine: - '*UserInitMprLogonScript*' - condition: (exec_selection and not exec_exclusion) or (create_selection_reg and create_keywords_reg) or (create_selection_cli and create_keywords_cli) + condition: (exec_selection and not exec_exclusion1 and not exec_exclusion2) or (create_selection_reg and create_keywords_reg) or (create_selection_cli and create_keywords_cli) falsepositives: - exclude legitimate logon scripts - penetration tests, red teaming diff --git a/rules/windows/sysmon/sysmon_susp_file_characteristics.yml b/rules/windows/sysmon/sysmon_susp_file_characteristics.yml index 19956fce5..d55dac169 100644 --- a/rules/windows/sysmon/sysmon_susp_file_characteristics.yml +++ b/rules/windows/sysmon/sysmon_susp_file_characteristics.yml @@ -6,6 +6,7 @@ references: - https://www.virustotal.com/#/file/276a765a10f98cda1a38d3a31e7483585ca3722ecad19d784441293acf1b7beb/detection author: Markus Neis date: 2018/11/22 +modified: 2019/11/09 tags: - attack.defense_evasion - attack.execution @@ -29,4 +30,4 @@ fields: - ParentCommandLine falsepositives: - Unknown -level: high +level: medium diff --git a/tools/config/generic/windows-audit.yml b/tools/config/generic/windows-audit.yml index dc3691201..83b143c96 100644 --- a/tools/config/generic/windows-audit.yml +++ b/tools/config/generic/windows-audit.yml @@ -12,4 +12,3 @@ logsources: fieldmappings: Image: NewProcessName ParentImage: ParentProcessName - CommandLine: ProcessCommandLine diff --git a/tools/setup.py b/tools/setup.py index 110b9aff6..8059c88e6 100644 --- a/tools/setup.py +++ b/tools/setup.py @@ -13,7 +13,7 @@ with open(path.join(here, 'README.md'), encoding='utf-8') as f: setup( name='sigmatools', - version='0.13', + version='0.14', description='Tools for the Generic Signature Format for SIEM Systems', long_description=long_description, long_description_content_type="text/markdown", @@ -36,7 +36,7 @@ setup( keywords='security monitoring siem logging signatures elasticsearch splunk ids sysmon', packages=['sigma', 'sigma.backends', 'sigma.config', 'sigma.parser', 'sigma.parser.modifiers'], python_requires='~=3.6', - install_requires=['PyYAML', 'pymisp'], + install_requires=['PyYAML', 'pymisp', 'progressbar2'], extras_require={ 'test': ['coverage', 'yamllint'], }, @@ -70,5 +70,6 @@ setup( 'sigmac', 'merge_sigma', 'sigma2misp', + 'sigma-similarity', ] ) diff --git a/tools/sigma/backends/elasticsearch.py b/tools/sigma/backends/elasticsearch.py index 1a7be9a3d..423b93dc4 100644 --- a/tools/sigma/backends/elasticsearch.py +++ b/tools/sigma/backends/elasticsearch.py @@ -212,8 +212,6 @@ class ElasticsearchDSLBackend(RulenameCommentMixin, ElasticsearchWildcardHandlin def generateMapItemNode(self, node): key, value = node - if type(value) not in (str, int, list, type(None)): - raise TypeError("Map values must be strings, numbers, lists or null, not " + str(type(value))) if type(value) is list: res = {'bool': {'should': []}} for v in value: @@ -230,7 +228,7 @@ class ElasticsearchDSLBackend(RulenameCommentMixin, ElasticsearchWildcardHandlin elif value is None: key_mapped = self.fieldNameMapping(key, value) return { "bool": { "must_not": { "exists": { "field": key_mapped } } } } - else: + elif type(value) in (str, int): key_mapped = self.fieldNameMapping(key, value) if self.matchKeyword: # searches against keyowrd fields are wildcard searches, phrases otherwise queryType = 'wildcard' @@ -239,6 +237,11 @@ class ElasticsearchDSLBackend(RulenameCommentMixin, ElasticsearchWildcardHandlin queryType = 'match_phrase' value_cleaned = self.cleanValue(str(value)) return {queryType: {key_mapped: value_cleaned}} + elif isinstance(value, SigmaRegularExpressionModifier): + key_mapped = self.fieldNameMapping(key, value) + return { 'regexp': { key_mapped: str(value) } } + else: + raise TypeError("Map values must be strings, numbers, lists, null or regular expression, not " + str(type(value))) def generateValueNode(self, node): return {'multi_match': {'query': node, 'fields': [], 'type': 'phrase'}}