From 9ce84a38e592f6d2163ca5aac90533bca4853981 Mon Sep 17 00:00:00 2001 From: pdr9rc Date: Wed, 29 Apr 2020 20:36:45 +0100 Subject: [PATCH 01/12] overrides section support + one example rule + cloudtrail config ditto --- rules/cloud/aws_ec2_vm_export_failure.yml | 28 +++++++++++ tools/config/ecs-cloudtrail.yml | 57 +++++++++++++++++++++++ tools/sigma/backends/base.py | 8 ++++ 3 files changed, 93 insertions(+) create mode 100644 rules/cloud/aws_ec2_vm_export_failure.yml create mode 100644 tools/config/ecs-cloudtrail.yml diff --git a/rules/cloud/aws_ec2_vm_export_failure.yml b/rules/cloud/aws_ec2_vm_export_failure.yml new file mode 100644 index 000000000..a8a80763b --- /dev/null +++ b/rules/cloud/aws_ec2_vm_export_failure.yml @@ -0,0 +1,28 @@ +title: AWS EC2 VM Export failure +id: 54b9a76a-3c71-4673-b4b3-2edb4566ea7b +status: experimental +description: An attempt to export an AWS EC2 instance has been detected. A VM Export might indicate an attempt to extract information from an instance. +references: + - https://docs.aws.amazon.com/vm-import/latest/userguide/vmexport.html#export-instance +author: Diogo Braz +date: 2020/04/16 +tags: + - attack.collection + - attack.t1005 + - attack.exfiltration + - attack.t1537 +level: low +logsource: + service: cloudtrail +detection: + selection: + eventName: 'CreateInstanceExportTask' + eventSource: 'ec2.amazonaws.com' + filter1: + errorMessage: '*' + filter2: + errorCode: '*' + filter3: + eventName: 'ConsoleLogin' + responseElements: '*Failure*' + condition: selection and (filter1 or filter2 or filter3) \ No newline at end of file diff --git a/tools/config/ecs-cloudtrail.yml b/tools/config/ecs-cloudtrail.yml new file mode 100644 index 000000000..e0a6b5818 --- /dev/null +++ b/tools/config/ecs-cloudtrail.yml @@ -0,0 +1,57 @@ +title: Elastic Common Schema mapping for cloudtrail logs +order: 20 +backends: + - es-qs + - es-dsl + - es-rule + - kibana + - xpack-watcher + - elastalert + - elastalert-dsl +fieldmappings: + additionalEventdata: aws.cloudtrail.additional_eventdata + apiVersion: aws.cloudtrail.api_version + awsRegion: cloud.region + errorCode: aws.cloudtrail.error_code + errorMessage: aws.cloudtrail.error_message + eventID: event.id + eventName: event.action + eventSource: event.provider + eventTime: '@timestamp' + eventType: aws.cloudtrail.event_type + eventVersion: aws.cloudtrail.event_version + managementEvent: aws.cloudtrail.management_event + readOnly: aws.cloudtrail.read_only + requestID: aws.cloudtrail.request_id + requestParameters: aws.cloudtrail.request_parameters + resources.accountId: aws.cloudtrail.resources.account_id + resources.ARN: aws.cloudtrail.resources.arn + resources.type: aws.cloudtrail.resources.type + responseElements: aws.cloudtrail.response_elements + serviceEventDetails: aws.cloudtrail.service_event_details + sharedEventId: aws.cloudtrail.shared_event_id + sourceIPAddress: source.address + userAgent: user_agent + userIdentity.accessKeyId: aws.cloudtrail.user_identity.access_key_id + userIdentity.accountId: cloud.account.id + userIdentity.arn: aws.cloudtrail.user_identity.arn + userIdentity.invokedBy: aws.cloudtrail.user_identity.invoked_by + userIdentity.principalId: user.id + userIdentity.sessionContext.attributes.creationDate: aws.cloudtrail.user_identity.session_context.creation_date + userIdentity.sessionContext.attributes.mfaAuthenticated: aws.cloudtrail.user_identity.session_context.mfa_authenticated + userIdentity.type: aws.cloudtrail.user_identity.type + userIdentity.userName: user.name + vpcEndpointId: aws.cloudtrail.vpc_endpoint_id +overrides: + - field: event_outcome_errors + value: '' + regexes: + - \b(aws.cloudtrail.error_message.keyword:.* OR aws.cloudtrail.error_code.keyword:.*|aws.cloudtrail.error_code.keyword:.* OR aws.cloudtrail.error_message.keyword:.*)\b + - field: event_outcome_login + value: '' + regexes: + - \b(event.action:"ConsoleLogin" AND aws.cloudtrail.response_elements.keyword:.*Failure|aws.cloudtrail.response_elements.keyword:.*Failure AND event.action:"ConsoleLogin")\b + - field: event.outcome + value: failure + regexes: + - '\b(event_outcome_errors: OR event_outcome_login:|event_outcome_login: OR event_outcome_errors:)' \ No newline at end of file diff --git a/tools/sigma/backends/base.py b/tools/sigma/backends/base.py index 4675b0197..1bd94df63 100644 --- a/tools/sigma/backends/base.py +++ b/tools/sigma/backends/base.py @@ -18,6 +18,7 @@ import sys import sigma import yaml +import re from .mixins import RulenameCommentMixin, QuoteCharMixin from sigma.parser.modifiers.base import SigmaTypeModifier @@ -90,6 +91,7 @@ class BaseBackend: options = tuple() # a list of tuples with following elements: option name, default value, help text, target attribute name (option name if None) config_required = True default_config = None + mapExpression = "" def __init__(self, sigmaconfig, backend_options=dict()): """ @@ -130,6 +132,12 @@ class BaseBackend: result = self.generateNode(parsed.parsedSearch) if parsed.parsedAgg: result += self.generateAggregation(parsed.parsedAgg) + if 'overrides' in self.sigmaconfig.config: + for expression in self.sigmaconfig.config['overrides']: + for x in expression['regexes']: + sub = expression['field'] + value = expression['value'] + result = re.sub(x, self.mapExpression % (sub, value), result) return result def generateNode(self, node): From ac4a2b1f26df3071af13030731bb384290bf2422 Mon Sep 17 00:00:00 2001 From: pdr9rc Date: Wed, 29 Apr 2020 22:55:46 +0100 Subject: [PATCH 02/12] wip wip --- tools/config/ecs-cloudtrail.yml | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/tools/config/ecs-cloudtrail.yml b/tools/config/ecs-cloudtrail.yml index e0a6b5818..cde889b20 100644 --- a/tools/config/ecs-cloudtrail.yml +++ b/tools/config/ecs-cloudtrail.yml @@ -43,15 +43,14 @@ fieldmappings: userIdentity.userName: user.name vpcEndpointId: aws.cloudtrail.vpc_endpoint_id overrides: - - field: event_outcome_errors - value: '' - regexes: - - \b(aws.cloudtrail.error_message.keyword:.* OR aws.cloudtrail.error_code.keyword:.*|aws.cloudtrail.error_code.keyword:.* OR aws.cloudtrail.error_message.keyword:.*)\b - - field: event_outcome_login - value: '' - regexes: - - \b(event.action:"ConsoleLogin" AND aws.cloudtrail.response_elements.keyword:.*Failure|aws.cloudtrail.response_elements.keyword:.*Failure AND event.action:"ConsoleLogin")\b - - field: event.outcome + - field: event_outcome value: failure - regexes: - - '\b(event_outcome_errors: OR event_outcome_login:|event_outcome_login: OR event_outcome_errors:)' \ No newline at end of file + regexes: + - (\(aws.cloudtrail.error_message.keyword:.* event.action:\"ConsoleLogin\"\)) + - (\(aws.cloudtrail.error_code.keyword:.* event.action:\"ConsoleLogin\"\)) + - (\(aws.cloudtrail.error_message.keyword:.* aws.cloudtrail.response_elements.keyword:\*Failure\*\)) + - (\(aws.cloudtrail.error_code.keyword:.* aws.cloudtrail.response_elements.keyword:\*Failure\*\)) + - (\(event.action:\"ConsoleLogin\".* aws.cloudtrail.error_message.keyword:\*\)) + - (\(event.action:\"ConsoleLogin\".* aws.cloudtrail.error_code.keyword:\*\)) + - (\(aws.cloudtrail.response_elements.keyword:\*Failure\*.* aws.cloudtrail.error_message.keyword:\*\)) + - (\(aws.cloudtrail.response_elements.keyword:\*Failure\*.* aws.cloudtrail.error_code.keyword:\*\)) \ No newline at end of file From dfdb5b9550794f0df4bf6015d901226671586be5 Mon Sep 17 00:00:00 2001 From: Tiago Faria Date: Wed, 29 Apr 2020 23:59:26 +0100 Subject: [PATCH 03/12] better description and event.outcome --- tools/config/ecs-cloudtrail.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/config/ecs-cloudtrail.yml b/tools/config/ecs-cloudtrail.yml index cde889b20..a3ea6851b 100644 --- a/tools/config/ecs-cloudtrail.yml +++ b/tools/config/ecs-cloudtrail.yml @@ -1,4 +1,4 @@ -title: Elastic Common Schema mapping for cloudtrail logs +title: Elastic Common Schema and Elastic Exported Fields mapping for AWS CloudTrail logs order: 20 backends: - es-qs @@ -43,7 +43,7 @@ fieldmappings: userIdentity.userName: user.name vpcEndpointId: aws.cloudtrail.vpc_endpoint_id overrides: - - field: event_outcome + - field: event.outcome value: failure regexes: - (\(aws.cloudtrail.error_message.keyword:.* event.action:\"ConsoleLogin\"\)) @@ -53,4 +53,4 @@ overrides: - (\(event.action:\"ConsoleLogin\".* aws.cloudtrail.error_message.keyword:\*\)) - (\(event.action:\"ConsoleLogin\".* aws.cloudtrail.error_code.keyword:\*\)) - (\(aws.cloudtrail.response_elements.keyword:\*Failure\*.* aws.cloudtrail.error_message.keyword:\*\)) - - (\(aws.cloudtrail.response_elements.keyword:\*Failure\*.* aws.cloudtrail.error_code.keyword:\*\)) \ No newline at end of file + - (\(aws.cloudtrail.response_elements.keyword:\*Failure\*.* aws.cloudtrail.error_code.keyword:\*\)) From 8142244449efd6c62953a8f05e5d8256910ab358 Mon Sep 17 00:00:00 2001 From: pdr9rc Date: Thu, 30 Apr 2020 15:08:20 +0100 Subject: [PATCH 04/12] wip wip --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index bf7103a4d..131863727 100644 --- a/.gitignore +++ b/.gitignore @@ -94,3 +94,4 @@ settings.json # VisualStudio .vs/ +.vscode/launch.json From 98391f985a17f7a1a694857468ad3b66a2515025 Mon Sep 17 00:00:00 2001 From: pdr9rc Date: Thu, 30 Apr 2020 15:19:38 +0100 Subject: [PATCH 05/12] wip wip --- tools/config/ecs-cloudtrail.yml | 20 ++++++++++++-------- tools/sigma/backends/base.py | 14 ++++++++++---- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/tools/config/ecs-cloudtrail.yml b/tools/config/ecs-cloudtrail.yml index a3ea6851b..96e45d1d8 100644 --- a/tools/config/ecs-cloudtrail.yml +++ b/tools/config/ecs-cloudtrail.yml @@ -46,11 +46,15 @@ overrides: - field: event.outcome value: failure regexes: - - (\(aws.cloudtrail.error_message.keyword:.* event.action:\"ConsoleLogin\"\)) - - (\(aws.cloudtrail.error_code.keyword:.* event.action:\"ConsoleLogin\"\)) - - (\(aws.cloudtrail.error_message.keyword:.* aws.cloudtrail.response_elements.keyword:\*Failure\*\)) - - (\(aws.cloudtrail.error_code.keyword:.* aws.cloudtrail.response_elements.keyword:\*Failure\*\)) - - (\(event.action:\"ConsoleLogin\".* aws.cloudtrail.error_message.keyword:\*\)) - - (\(event.action:\"ConsoleLogin\".* aws.cloudtrail.error_code.keyword:\*\)) - - (\(aws.cloudtrail.response_elements.keyword:\*Failure\*.* aws.cloudtrail.error_message.keyword:\*\)) - - (\(aws.cloudtrail.response_elements.keyword:\*Failure\*.* aws.cloudtrail.error_code.keyword:\*\)) + - (\(\(aws.cloudtrail.error_message.keyword:.* event.action:\"ConsoleLogin\"\)\)) + - (\(\(aws.cloudtrail.error_code.keyword:.* event.action:\"ConsoleLogin\"\)\)) + - (\(\(aws.cloudtrail.error_message.keyword:.* aws.cloudtrail.response_elements.keyword:\*Failure\*\)\)) + - (\(\(aws.cloudtrail.error_code.keyword:.* aws.cloudtrail.response_elements.keyword:\*Failure\*\)\)) + - (\(\(event.action:\"ConsoleLogin\".* aws.cloudtrail.error_message.keyword:\*\)\)) + - (\(\(event.action:\"ConsoleLogin\".* aws.cloudtrail.error_code.keyword:\*\)\)) + - (\(\(aws.cloudtrail.response_elements.keyword:\*Failure\*.* aws.cloudtrail.error_message.keyword:\*\)\)) + - (\(\(aws.cloudtrail.response_elements.keyword:\*Failure\*.* aws.cloudtrail.error_code.keyword:\*\)\)) + - field: event.outcome + value: success + literals: + - 'NOT (event.outcome:failure)' \ No newline at end of file diff --git a/tools/sigma/backends/base.py b/tools/sigma/backends/base.py index 1bd94df63..3e27c1246 100644 --- a/tools/sigma/backends/base.py +++ b/tools/sigma/backends/base.py @@ -134,10 +134,16 @@ class BaseBackend: result += self.generateAggregation(parsed.parsedAgg) if 'overrides' in self.sigmaconfig.config: for expression in self.sigmaconfig.config['overrides']: - for x in expression['regexes']: - sub = expression['field'] - value = expression['value'] - result = re.sub(x, self.mapExpression % (sub, value), result) + if 'regexes' in expression: + for x in expression['regexes']: + sub = expression['field'] + value = expression['value'] + result = re.sub(x, self.mapExpression % (sub, value), result) + if 'literals' in expression: + for x in expression['literals']: + sub = expression['field'] + value = expression['value'] + result = result.replace(x, self.mapExpression % (sub, value)) return result def generateNode(self, node): From bc0a2c7ab932cf967869cc31237b18b1cef0e09e Mon Sep 17 00:00:00 2001 From: pdr9rc Date: Fri, 1 May 2020 19:20:05 +0100 Subject: [PATCH 06/12] wip wip --- tools/config/ecs-cloudtrail.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tools/config/ecs-cloudtrail.yml b/tools/config/ecs-cloudtrail.yml index 96e45d1d8..37414528b 100644 --- a/tools/config/ecs-cloudtrail.yml +++ b/tools/config/ecs-cloudtrail.yml @@ -54,6 +54,15 @@ overrides: - (\(\(event.action:\"ConsoleLogin\".* aws.cloudtrail.error_code.keyword:\*\)\)) - (\(\(aws.cloudtrail.response_elements.keyword:\*Failure\*.* aws.cloudtrail.error_message.keyword:\*\)\)) - (\(\(aws.cloudtrail.response_elements.keyword:\*Failure\*.* aws.cloudtrail.error_code.keyword:\*\)\)) + literals: + - ((aws.cloudtrail.error_message.keyword:* OR aws.cloudtrail.error_code.keyword:*) OR (event.action:"ConsoleLogin" AND aws.cloudtrail.response_elements.keyword:*Failure*)) + - ((aws.cloudtrail.error_code.keyword:* OR aws.cloudtrail.error_message.keyword:*) OR (event.action:"ConsoleLogin" AND aws.cloudtrail.response_elements.keyword:*Failure*)) + - ((aws.cloudtrail.error_message.keyword:* OR aws.cloudtrail.error_code.keyword:*) OR (aws.cloudtrail.response_elements.keyword:*Failure* AND event.action:"ConsoleLogin")) + - ((aws.cloudtrail.error_code.keyword:* OR aws.cloudtrail.error_message.keyword:*) OR (aws.cloudtrail.response_elements.keyword:*Failure* AND event.action:"ConsoleLogin")) + - ((event.action:"ConsoleLogin" AND aws.cloudtrail.response_elements.keyword:*Failure*) OR (aws.cloudtrail.error_message.keyword:* OR aws.cloudtrail.error_code.keyword:*)) + - ((event.action:"ConsoleLogin" AND aws.cloudtrail.response_elements.keyword:*Failure*) OR (aws.cloudtrail.error_code.keyword:* OR aws.cloudtrail.error_message.keyword:*)) + - ((aws.cloudtrail.response_elements.keyword:*Failure* AND event.action:"ConsoleLogin") OR (aws.cloudtrail.error_message.keyword:* OR aws.cloudtrail.error_code.keyword:*)) + - ((aws.cloudtrail.response_elements.keyword:*Failure* AND event.action:"ConsoleLogin") OR (aws.cloudtrail.error_code.keyword:* OR aws.cloudtrail.error_message.keyword:*)) - field: event.outcome value: success literals: From dd85467a271d79ec30079e0e4b6391e1a465cd0a Mon Sep 17 00:00:00 2001 From: Tiago Faria Date: Sat, 2 May 2020 00:13:55 +0100 Subject: [PATCH 07/12] Update aws_ec2_vm_export_failure.yml --- rules/cloud/aws_ec2_vm_export_failure.yml | 34 +++++++++++------------ 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/rules/cloud/aws_ec2_vm_export_failure.yml b/rules/cloud/aws_ec2_vm_export_failure.yml index a8a80763b..8f7fec195 100644 --- a/rules/cloud/aws_ec2_vm_export_failure.yml +++ b/rules/cloud/aws_ec2_vm_export_failure.yml @@ -3,26 +3,26 @@ id: 54b9a76a-3c71-4673-b4b3-2edb4566ea7b status: experimental description: An attempt to export an AWS EC2 instance has been detected. A VM Export might indicate an attempt to extract information from an instance. references: - - https://docs.aws.amazon.com/vm-import/latest/userguide/vmexport.html#export-instance + - https://docs.aws.amazon.com/vm-import/latest/userguide/vmexport.html#export-instance author: Diogo Braz date: 2020/04/16 tags: - - attack.collection - - attack.t1005 - - attack.exfiltration - - attack.t1537 + - attack.collection + - attack.t1005 + - attack.exfiltration + - attack.t1537 level: low logsource: - service: cloudtrail + service: cloudtrail detection: - selection: - eventName: 'CreateInstanceExportTask' - eventSource: 'ec2.amazonaws.com' - filter1: - errorMessage: '*' - filter2: - errorCode: '*' - filter3: - eventName: 'ConsoleLogin' - responseElements: '*Failure*' - condition: selection and (filter1 or filter2 or filter3) \ No newline at end of file + selection: + eventName: 'CreateInstanceExportTask' + eventSource: 'ec2.amazonaws.com' + filter1: + errorMessage: '*' + filter2: + errorCode: '*' + filter3: + eventName: 'ConsoleLogin' + responseElements: '*Failure*' + condition: selection and (filter1 or filter2 or filter3) From b3194e66c4f1def35c11f407d6c7e47867eb8053 Mon Sep 17 00:00:00 2001 From: pdr9rc Date: Mon, 4 May 2020 16:37:36 +0100 Subject: [PATCH 08/12] Update base.py --- tools/sigma/backends/base.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/tools/sigma/backends/base.py b/tools/sigma/backends/base.py index 3e27c1246..dfc0d0288 100644 --- a/tools/sigma/backends/base.py +++ b/tools/sigma/backends/base.py @@ -132,19 +132,35 @@ class BaseBackend: result = self.generateNode(parsed.parsedSearch) if parsed.parsedAgg: result += self.generateAggregation(parsed.parsedAgg) + # if 'overrides' in self.sigmaconfig.config: + # for expression in self.sigmaconfig.config['overrides']: + # if 'regexes' in expression: + # for x in expression['regexes']: + # sub = expression['field'] + # value = expression['value'] + # result = re.sub(x, self.mapExpression % (sub, value), result) + # if 'literals' in expression: + # for x in expression['literals']: + # sub = expression['field'] + # value = expression['value'] + # result = result.replace(x, self.mapExpression % (sub, value)) + result = self.applyOverrides(result) + return result + + def applyOverrides(self, query): if 'overrides' in self.sigmaconfig.config: for expression in self.sigmaconfig.config['overrides']: if 'regexes' in expression: for x in expression['regexes']: sub = expression['field'] value = expression['value'] - result = re.sub(x, self.mapExpression % (sub, value), result) + query = re.sub(x, self.mapExpression % (sub, value), query) if 'literals' in expression: for x in expression['literals']: sub = expression['field'] value = expression['value'] - result = result.replace(x, self.mapExpression % (sub, value)) - return result + query = query.replace(x, self.mapExpression % (sub, value)) + return query def generateNode(self, node): if type(node) == sigma.parser.condition.ConditionAND: From dd9e128a15d5f9ad08ea744cde0f18b49a4a1204 Mon Sep 17 00:00:00 2001 From: pdr9rc Date: Mon, 4 May 2020 17:35:12 +0100 Subject: [PATCH 09/12] kibana target update kibana target now compatible with overrides --- tools/sigma/backends/base.py | 12 ------------ tools/sigma/backends/elasticsearch.py | 2 +- 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/tools/sigma/backends/base.py b/tools/sigma/backends/base.py index dfc0d0288..b4003f5a1 100644 --- a/tools/sigma/backends/base.py +++ b/tools/sigma/backends/base.py @@ -132,18 +132,6 @@ class BaseBackend: result = self.generateNode(parsed.parsedSearch) if parsed.parsedAgg: result += self.generateAggregation(parsed.parsedAgg) - # if 'overrides' in self.sigmaconfig.config: - # for expression in self.sigmaconfig.config['overrides']: - # if 'regexes' in expression: - # for x in expression['regexes']: - # sub = expression['field'] - # value = expression['value'] - # result = re.sub(x, self.mapExpression % (sub, value), result) - # if 'literals' in expression: - # for x in expression['literals']: - # sub = expression['field'] - # value = expression['value'] - # result = result.replace(x, self.mapExpression % (sub, value)) result = self.applyOverrides(result) return result diff --git a/tools/sigma/backends/elasticsearch.py b/tools/sigma/backends/elasticsearch.py index 397ff9438..080bd5ad6 100644 --- a/tools/sigma/backends/elasticsearch.py +++ b/tools/sigma/backends/elasticsearch.py @@ -566,7 +566,7 @@ class KibanaBackend(ElasticsearchQuerystringBackend, MultiRuleOutputMixin): }, "query": { "query_string": { - "query": result, + "query": self.applyOverrides(result), "analyze_wildcard": True } } From aa175a7d5bbfd8b9e6526cfa36256dac5514bff4 Mon Sep 17 00:00:00 2001 From: pdr9rc Date: Mon, 4 May 2020 18:02:27 +0100 Subject: [PATCH 10/12] wip wip --- tools/sigma/backends/base.py | 49 ++++++++++++++------------- tools/sigma/backends/elasticsearch.py | 2 +- 2 files changed, 27 insertions(+), 24 deletions(-) diff --git a/tools/sigma/backends/base.py b/tools/sigma/backends/base.py index b4003f5a1..d4c7ad55f 100644 --- a/tools/sigma/backends/base.py +++ b/tools/sigma/backends/base.py @@ -132,45 +132,48 @@ class BaseBackend: result = self.generateNode(parsed.parsedSearch) if parsed.parsedAgg: result += self.generateAggregation(parsed.parsedAgg) - result = self.applyOverrides(result) + #result = self.applyOverrides(result) return result def applyOverrides(self, query): - if 'overrides' in self.sigmaconfig.config: - for expression in self.sigmaconfig.config['overrides']: - if 'regexes' in expression: - for x in expression['regexes']: - sub = expression['field'] - value = expression['value'] - query = re.sub(x, self.mapExpression % (sub, value), query) - if 'literals' in expression: - for x in expression['literals']: - sub = expression['field'] - value = expression['value'] - query = query.replace(x, self.mapExpression % (sub, value)) + try: + if 'overrides' in self.sigmaconfig.config and isinstance(query, str): + for expression in self.sigmaconfig.config['overrides']: + if 'regexes' in expression: + for x in expression['regexes']: + sub = expression['field'] + value = expression['value'] + query = re.sub(x, self.mapExpression % (sub, value), query) + if 'literals' in expression: + for x in expression['literals']: + sub = expression['field'] + value = expression['value'] + query = query.replace(x, self.mapExpression % (sub, value)) + except Exception: + pass return query def generateNode(self, node): if type(node) == sigma.parser.condition.ConditionAND: - return self.generateANDNode(node) + return self.applyOverrides(self.generateANDNode(node)) elif type(node) == sigma.parser.condition.ConditionOR: - return self.generateORNode(node) + return self.applyOverrides(self.generateORNode(node)) elif type(node) == sigma.parser.condition.ConditionNOT: - return self.generateNOTNode(node) + return self.applyOverrides(self.generateNOTNode(node)) elif type(node) == sigma.parser.condition.ConditionNULLValue: - return self.generateNULLValueNode(node) + return self.applyOverrides(self.generateNULLValueNode(node)) elif type(node) == sigma.parser.condition.ConditionNotNULLValue: - return self.generateNotNULLValueNode(node) + return self.applyOverrides(self.generateNotNULLValueNode(node)) elif type(node) == sigma.parser.condition.NodeSubexpression: - return self.generateSubexpressionNode(node) + return self.applyOverrides(self.generateSubexpressionNode(node)) elif type(node) == tuple: - return self.generateMapItemNode(node) + return self.applyOverrides(self.generateMapItemNode(node)) elif type(node) in (str, int): - return self.generateValueNode(node) + return self.applyOverrides(self.generateValueNode(node)) elif type(node) == list: - return self.generateListNode(node) + return self.applyOverrides(self.generateListNode(node)) elif isinstance(node, SigmaTypeModifier): - return self.generateTypedValueNode(node) + return self.applyOverrides(self.generateTypedValueNode(node)) else: raise TypeError("Node type %s was not expected in Sigma parse tree" % (str(type(node)))) diff --git a/tools/sigma/backends/elasticsearch.py b/tools/sigma/backends/elasticsearch.py index 080bd5ad6..397ff9438 100644 --- a/tools/sigma/backends/elasticsearch.py +++ b/tools/sigma/backends/elasticsearch.py @@ -566,7 +566,7 @@ class KibanaBackend(ElasticsearchQuerystringBackend, MultiRuleOutputMixin): }, "query": { "query_string": { - "query": self.applyOverrides(result), + "query": result, "analyze_wildcard": True } } From 31ad81874fee378ea3f951d81d088aac9fb1e737 Mon Sep 17 00:00:00 2001 From: pdr9rc Date: Tue, 5 May 2020 11:32:18 +0100 Subject: [PATCH 11/12] capitalized titles corrected capitalization of titles and removed literals from config --- rules/cloud/aws_ec2_vm_export_failure.yml | 2 +- tools/config/ecs-cloudtrail.yml | 11 +---------- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/rules/cloud/aws_ec2_vm_export_failure.yml b/rules/cloud/aws_ec2_vm_export_failure.yml index 8f7fec195..a6db628c5 100644 --- a/rules/cloud/aws_ec2_vm_export_failure.yml +++ b/rules/cloud/aws_ec2_vm_export_failure.yml @@ -1,4 +1,4 @@ -title: AWS EC2 VM Export failure +title: AWS EC2 VM Export Failure id: 54b9a76a-3c71-4673-b4b3-2edb4566ea7b status: experimental description: An attempt to export an AWS EC2 instance has been detected. A VM Export might indicate an attempt to extract information from an instance. diff --git a/tools/config/ecs-cloudtrail.yml b/tools/config/ecs-cloudtrail.yml index 37414528b..fe9419bd4 100644 --- a/tools/config/ecs-cloudtrail.yml +++ b/tools/config/ecs-cloudtrail.yml @@ -1,4 +1,4 @@ -title: Elastic Common Schema and Elastic Exported Fields mapping for AWS CloudTrail logs +title: Elastic Common Schema And Elastic Exported Fields Mapping For AWS CloudTrail Logs order: 20 backends: - es-qs @@ -54,15 +54,6 @@ overrides: - (\(\(event.action:\"ConsoleLogin\".* aws.cloudtrail.error_code.keyword:\*\)\)) - (\(\(aws.cloudtrail.response_elements.keyword:\*Failure\*.* aws.cloudtrail.error_message.keyword:\*\)\)) - (\(\(aws.cloudtrail.response_elements.keyword:\*Failure\*.* aws.cloudtrail.error_code.keyword:\*\)\)) - literals: - - ((aws.cloudtrail.error_message.keyword:* OR aws.cloudtrail.error_code.keyword:*) OR (event.action:"ConsoleLogin" AND aws.cloudtrail.response_elements.keyword:*Failure*)) - - ((aws.cloudtrail.error_code.keyword:* OR aws.cloudtrail.error_message.keyword:*) OR (event.action:"ConsoleLogin" AND aws.cloudtrail.response_elements.keyword:*Failure*)) - - ((aws.cloudtrail.error_message.keyword:* OR aws.cloudtrail.error_code.keyword:*) OR (aws.cloudtrail.response_elements.keyword:*Failure* AND event.action:"ConsoleLogin")) - - ((aws.cloudtrail.error_code.keyword:* OR aws.cloudtrail.error_message.keyword:*) OR (aws.cloudtrail.response_elements.keyword:*Failure* AND event.action:"ConsoleLogin")) - - ((event.action:"ConsoleLogin" AND aws.cloudtrail.response_elements.keyword:*Failure*) OR (aws.cloudtrail.error_message.keyword:* OR aws.cloudtrail.error_code.keyword:*)) - - ((event.action:"ConsoleLogin" AND aws.cloudtrail.response_elements.keyword:*Failure*) OR (aws.cloudtrail.error_code.keyword:* OR aws.cloudtrail.error_message.keyword:*)) - - ((aws.cloudtrail.response_elements.keyword:*Failure* AND event.action:"ConsoleLogin") OR (aws.cloudtrail.error_message.keyword:* OR aws.cloudtrail.error_code.keyword:*)) - - ((aws.cloudtrail.response_elements.keyword:*Failure* AND event.action:"ConsoleLogin") OR (aws.cloudtrail.error_code.keyword:* OR aws.cloudtrail.error_message.keyword:*)) - field: event.outcome value: success literals: From 06abd6e76a0487ac0bbe414ecae9ce021bce4106 Mon Sep 17 00:00:00 2001 From: Tiago Faria Date: Thu, 14 May 2020 14:03:23 +0100 Subject: [PATCH 12/12] added ci tests for ecs-cloudtrail --- Makefile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Makefile b/Makefile index 1ad713518..18a3dbb74 100644 --- a/Makefile +++ b/Makefile @@ -31,6 +31,11 @@ test-sigmac: $(COVERAGE) run -a --include=$(COVSCOPE) tools/sigmac -rvdI -t elastalert -c tools/config/winlogbeat.yml -O alert_methods=http_post,email -O emails=test@test.invalid -O http_post_url=http://test.invalid rules/ > /dev/null $(COVERAGE) run -a --include=$(COVSCOPE) tools/sigmac -rvdI -t elastalert-dsl -c tools/config/winlogbeat.yml -O alert_methods=http_post,email -O emails=test@test.invalid -O http_post_url=http://test.invalid rules/ > /dev/null $(COVERAGE) run -a --include=$(COVSCOPE) tools/sigmac -rvdI -t ee-outliers -c tools/config/winlogbeat.yml rules/ > /dev/null + $(COVERAGE) run -a --include=$(COVSCOPE) tools/sigmac -rvdI -t es-qs -c tools/config/ecs-cloudtrail.yml rules/ > /dev/null + $(COVERAGE) run -a --include=$(COVSCOPE) tools/sigmac -rvdI -t es-rule -c tools/config/ecs-cloudtrail.yml rules/ > /dev/null + $(COVERAGE) run -a --include=$(COVSCOPE) tools/sigmac -rvdI -t kibana -c tools/config/ecs-cloudtrail.yml rules/ > /dev/null + $(COVERAGE) run -a --include=$(COVSCOPE) tools/sigmac -rvdI -t xpack-watcher -c tools/config/ecs-cloudtrail.yml rules/ > /dev/null + $(COVERAGE) run -a --include=$(COVSCOPE) tools/sigmac -rvdI -t elastalert -c tools/config/ecs-cloudtrail.yml rules/ > /dev/null ! $(COVERAGE) run -a --include=$(COVSCOPE) tools/sigmac -rvdI -t splunk rules/ > /dev/null $(COVERAGE) run -a --include=$(COVSCOPE) tools/sigmac -rvdI -t splunk -c tools/config/splunk-windows-index.yml rules/ > /dev/null $(COVERAGE) run -a --include=$(COVSCOPE) tools/sigmac -rvdI -t splunkxml -c tools/config/splunk-windows.yml rules/ > /dev/null