[Rule Tuning] Add Investigation Fields to Specific AWS Rules (#4261)

* adding investigation fields to specific aws rules

* updated patch

* removing min-stack requirements

* removed user.name redundancy

* adjusted order of investigation fields

* adding source address

detection_rules/etc/non-ecs-schema.json

@@ -152,7 +152,7 @@
     "aws.cloudtrail.flattened.request_parameters.fromPort": "keyword",
     "aws.cloudtrail.flattened.request_parameters.roleArn": "keyword",
     "aws.cloudtrail.flattened.request_parameters.roleName": "keyword",
-    "aws.cloudtrail.flattened.request_paramters.policyArn": "keyword",
+    "aws.cloudtrail.flattened.request_parameters.policyArn": "keyword",
     "aws.cloudtrail.flattened.request_parameters.serialNumber": "keyword"
   },
   "logs-azure.signinlogs-*": {

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "detection_rules"
-version = "0.1.5"
+version = "0.1.6"
 description = "Detection Rules is the home for rules used by Elastic Security. This repository is used for the development, maintenance, testing, validation, and release of rules for Elastic Security’s Detection Engine."
 readme = "README.md"
 requires-python = ">=3.12"

rules/cross-platform/execution_aws_ssm_sendcommand_with_command_parameters.toml

@@ -2,7 +2,7 @@
 creation_date = "2022/09/03"
 integration = ["endpoint", "auditd_manager"]
 maturity = "production"
-updated_date = "2024/09/23"
+updated_date = "2024/11/07"
 
 [rule]
 author = ["Elastic"]
@@ -87,6 +87,25 @@ and process.args: (
 )
 '''
 
+[rule.investigation_fields]
+field_names = [
+    "@timestamp",
+    "process.user.name",
+    "process.entry_leader.group.name",
+    "process.entry_leader.real_user.name",
+    "event.action",
+    "event.type",
+    "host.os.type",
+    "host.os.kernel",
+    "process.entry_leader.executable",
+    "process.entry_leader.working_directory",
+    "process.parent.executable",
+    "process.executable",
+    "process.hash.sha256",
+    "process.parent.command_line",
+    "process.command_line",
+    "process.args"
+]
 
 [[rule.threat]]
 framework = "MITRE ATT&CK"

rules/integrations/aws/defense_evasion_cloudtrail_logging_deleted.toml

@@ -2,7 +2,7 @@
 creation_date = "2020/05/26"
 integration = ["aws"]
 maturity = "production"
-updated_date = "2024/05/21"
+updated_date = "2024/11/07"
 
 [rule]
 author = ["Elastic"]
@@ -86,9 +86,26 @@ timestamp_override = "event.ingested"
 type = "query"
 
 query = '''
-event.dataset:aws.cloudtrail and event.provider:cloudtrail.amazonaws.com and event.action:DeleteTrail and event.outcome:success
+event.dataset:aws.cloudtrail
+    and event.provider:cloudtrail.amazonaws.com
+    and event.action:DeleteTrail
+    and event.outcome:success
 '''
 
+[rule.investigation_fields]
+field_names = [
+    "@timestamp",
+    "user.name",
+    "aws.cloudtrail.user_identity.arn",
+    "aws.cloudtrail.user_identity.type",
+    "source.address",
+    "user_agent.original",
+    "aws.cloudtrail.flattened.request_parameters.name",
+    "event.action",
+    "event.outcome",
+    "cloud.region",
+    "aws.cloudtrail.request_parameters"
+]
 
 [[rule.threat]]
 framework = "MITRE ATT&CK"

rules/integrations/aws/discovery_ec2_multi_region_describe_instances.toml

@@ -2,7 +2,7 @@
 creation_date = "2024/08/26"
 integration = ["aws"]
 maturity = "production"
-updated_date = "2024/11/05"
+updated_date = "2024/11/07"
 
 [rule]
 author = ["Elastic"]
@@ -107,6 +107,13 @@ from logs-aws.cloudtrail-*
 | sort target_time_window desc
 '''
 
+[rule.investigation_fields]
+field_names = [
+    "aws.cloudtrail.user_identity.arn",
+    "target_time_window",
+    "region_count",
+    "window_count"
+]
 
 [[rule.threat]]
 framework = "MITRE ATT&CK"

rules/integrations/aws/discovery_ec2_multiple_discovery_api_calls_via_cli.toml

@@ -4,7 +4,7 @@ integration = ["aws"]
 maturity = "production"
 min_stack_comments = "ES|QL rule type is still in technical preview as of 8.13, however this rule was tested successfully"
 min_stack_version = "8.13.0"
-updated_date = "2024/11/04"
+updated_date = "2024/11/07"
 
 [rule]
 author = ["Elastic"]
@@ -133,6 +133,12 @@ from logs-aws.cloudtrail*
 | sort unique_api_count desc
 '''
 
+[rule.investigation_fields]
+field_names = [
+    "time_window",
+    "aws.cloudtrail.user_identity.arn",
+    "unique_api_count"
+]
 
 [[rule.threat]]
 framework = "MITRE ATT&CK"

rules/integrations/aws/discovery_new_terms_sts_getcalleridentity.toml

@@ -2,19 +2,19 @@
 creation_date = "2024/05/24"
 integration = ["aws"]
 maturity = "production"
-updated_date = "2024/09/30"
+updated_date = "2024/11/07"
 
 [rule]
 author = ["Elastic"]
 description = """
-An adversary with access to a set of compromised credentials may attempt to verify that the credentials are valid and 
-determine what account they are using. This rule looks for the first time an identity has called the 
-STS `GetCallerIdentity` API operation in the last 15 days, which may be an indicator of compromised credentials. 
+An adversary with access to a set of compromised credentials may attempt to verify that the credentials are valid and
+determine what account they are using. This rule looks for the first time an identity has called the
+STS `GetCallerIdentity` API operation in the last 15 days, which may be an indicator of compromised credentials.
 A legitimate user would not need to call this operation as they should know the account they are using.
 """
 false_positives = [
     """
-    Verify whether the user identity should be using the STS `GetCallerIdentity` API operation. 
+    Verify whether the user identity should be using the STS `GetCallerIdentity` API operation.
     If known behavior is causing false positives, it can be exempted from the rule.
     """,
 ]
@@ -29,7 +29,7 @@ note = """## Triage and analysis
 ### Investigating AWS GetCallerIdentity API Called for the First Time
 
 AWS Security Token Service (AWS STS) is a service that enables you to request temporary, limited-privilege credentials for users.
-The `GetCallerIdentity` function returns details about the IAM user or role owning the credentials used to call the operation. 
+The `GetCallerIdentity` function returns details about the IAM user or role owning the credentials used to call the operation.
 No permissions are required to run this operation and the same information is returned even when access is denied.
 This rule looks for use of the `GetCallerIdentity` operation. This is a [New Terms](https://www.elastic.co/guide/en/security/master/rules-ui-create.html#create-new-terms-rule) rule indicating this is the first time a specific user identity has called this operation within the last 15 days.
 
@@ -93,10 +93,26 @@ timestamp_override = "event.ingested"
 type = "new_terms"
 
 query = '''
-event.dataset: "aws.cloudtrail" and event.provider: "sts.amazonaws.com" and event.action: "GetCallerIdentity"
-and not aws.cloudtrail.user_identity.type: "AssumedRole"
+event.dataset: "aws.cloudtrail"
+    and event.provider: "sts.amazonaws.com"
+    and event.action: "GetCallerIdentity"
+    and event.outcome: "success"
+    and not aws.cloudtrail.user_identity.type: "AssumedRole"
 '''
 
+[rule.investigation_fields]
+field_names = [
+    "@timestamp",
+    "user.name",
+    "source.address",
+    "aws.cloudtrail.user_identity.type",
+    "aws.cloudtrail.user_identity.arn",
+    "user_agent.original",
+    "event.action",
+    "event.outcome",
+    "cloud.region",
+    "aws.cloudtrail.request_parameters"
+]
 
 [[rule.threat]]
 framework = "MITRE ATT&CK"

rules/integrations/aws/execution_ssm_command_document_created_by_rare_user.toml

@@ -2,16 +2,19 @@
 creation_date = "2024/11/01"
 integration = ["aws"]
 maturity = "production"
-updated_date = "2024/11/01"
+updated_date = "2024/11/07"
 
 [rule]
 author = ["Elastic"]
 description = """
-Identifies when an AWS Systems Manager (SSM) command document is created by a user who does not typically perform this action. Adversaries may create SSM command documents to execute commands on managed instances, potentially leading to unauthorized access, command and control, data exfiltration and more.
+Identifies when an AWS Systems Manager (SSM) command document is created by a user who does not typically perform this
+action. Adversaries may create SSM command documents to execute commands on managed instances, potentially leading to
+unauthorized access, command and control, data exfiltration and more.
 """
 false_positives = [
     """
-    Legitimate users may create SSM command documents for legitimate purposes. Ensure that the document is authorized and the user is known before taking action.
+    Legitimate users may create SSM command documents for legitimate purposes. Ensure that the document is authorized
+    and the user is known before taking action.
     """,
 ]
 from = "now-9m"
@@ -19,8 +22,7 @@ index = ["filebeat-*", "logs-aws.cloudtrail-*"]
 language = "kuery"
 license = "Elastic License v2"
 name = "AWS SSM Command Document Created by Rare User"
-note = """
-## Triage and Analysis
+note = """## Triage and Analysis
 
 ### Investigating AWS SSM Command Document Created by Rare User
 
@@ -55,7 +57,7 @@ For further guidance on managing and securing AWS Systems Manager in your enviro
 """
 references = [
     "https://docs.aws.amazon.com/systems-manager/latest/APIReference/API_CreateDocument.html",
-    "https://docs.aws.amazon.com/systems-manager/latest/userguide/documents.html"
+    "https://docs.aws.amazon.com/systems-manager/latest/userguide/documents.html",
 ]
 risk_score = 21
 rule_id = "50a2bdea-9876-11ef-89db-f661ea17fbcd"
@@ -68,7 +70,7 @@ tags = [
     "Data Source: AWS Systems Manager",
     "Resources: Investigation Guide",
     "Use Case: Threat Detection",
-    "Tactic: Execution"
+    "Tactic: Execution",
 ]
 timestamp_override = "event.ingested"
 type = "new_terms"
@@ -81,6 +83,21 @@ event.dataset: "aws.cloudtrail"
     and aws.cloudtrail.response_elements: *documentType=Command*
 '''
 
+[rule.investigation_fields]
+field_names = [
+    "@timestamp",
+    "user.name",
+    "source.address",
+    "aws.cloudtrail.user_identity.arn",
+    "aws.cloudtrail.user_identity.type",
+    "user_agent.original",
+    "event.action",
+    "event.outcome",
+    "cloud.region",
+    "aws.cloudtrail.request_parameters",
+    "aws.cloudtrail.response_elements",
+]
+
 [[rule.threat]]
 framework = "MITRE ATT&CK"
 

rules/integrations/aws/exfiltration_sns_email_subscription_by_rare_user.toml

@@ -2,7 +2,7 @@
 creation_date = "2024/11/01"
 integration = ["aws"]
 maturity = "production"
-updated_date = "2024/11/01"
+updated_date = "2024/11/07"
 
 [rule]
 author = ["Elastic"]
@@ -81,6 +81,22 @@ event.dataset: "aws.cloudtrail"
     and aws.cloudtrail.request_parameters: *protocol=email*
 '''
 
+[rule.investigation_fields]
+field_names = [
+    "@timestamp",
+    "user.name",
+    "source.address",
+    "aws.cloudtrail.user_identity.arn",
+    "aws.cloudtrail.user_identity.type",
+    "user_agent.original",
+    "event.action",
+    "event.outcome",
+    "cloud.region",
+    "aws.cloudtrail.flattened.request_parameters.protocol",
+    "aws.cloudtrail.flattened.request_parameters.topicArn",
+    "aws.cloudtrail.flattened.response_elements.subscriptionArn",
+    "aws.cloudtrail.request_parameters"
+]
 
 [[rule.threat]]
 framework = "MITRE ATT&CK"

rules/integrations/aws/impact_aws_s3_bucket_enumeration_or_brute_force.toml

@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2024/05/01"
 maturity = "production"
-updated_date = "2024/10/09"
+updated_date = "2024/11/07"
 min_stack_comments = "ES|QL rule type is still in technical preview as of 8.13, however this rule was tested successfully"
 min_stack_version = "8.13.0"
 
@@ -97,6 +97,13 @@ from logs-aws.cloudtrail*
 | where failed_requests > 40
 '''
 
+[rule.investigation_fields]
+field_names = [
+    "source.address",
+    "tls.client.server_name",
+    "cloud.account.id",
+    "failed_requests"
+]
 
 [[rule.threat]]
 framework = "MITRE ATT&CK"

rules/integrations/aws/persistence_ec2_security_group_configuration_change_detection.toml

@@ -2,7 +2,7 @@
 creation_date = "2021/05/05"
 integration = ["aws"]
 maturity = "production"
-updated_date = "2024/11/05"
+updated_date = "2024/11/07"
 
 [rule]
 author = ["Elastic", "Austin Songer"]
@@ -100,6 +100,22 @@ event.dataset: "aws.cloudtrail"
     and event.outcome: "success"
 '''
 
+[rule.investigation_fields]
+field_names = [
+   "@timestamp",
+   "user.name",
+   "aws.cloudtrail.user_identity.arn",
+   "aws.cloudtrail.user_identity.type",
+   "user_agent.original",
+   "aws.cloudtrail.flattened.request_parameters.instanceId",
+   "event.action",
+   "event.outcome",
+   "cloud.region",
+   "event.provider",
+   "aws.cloudtrail.request_parameters",
+   "aws.cloudtrail.response_elements"
+]
+
 
 [[rule.threat]]
 framework = "MITRE ATT&CK"

rules/integrations/aws/persistence_iam_create_user_via_assumed_role_on_ec2_instance.toml

@@ -2,7 +2,7 @@
 creation_date = "2024/11/04"
 integration = ["aws"]
 maturity = "production"
-updated_date = "2024/11/04"
+updated_date = "2024/11/07"
 
 [rule]
 author = ["Elastic"]
@@ -96,6 +96,20 @@ event.dataset: "aws.cloudtrail"
     and aws.cloudtrail.user_identity.arn: *i-*
 '''
 
+[rule.investigation_fields]
+field_names = [
+    "@timestamp",
+    "user.name",
+    "source.address",
+    "aws.cloudtrail.user_identity.arn",
+    "aws.cloudtrail.user_identity.type",
+    "user_agent.original",
+    "event.action",
+    "event.outcome",
+    "cloud.region",
+    "aws.cloudtrail.request_parameters",
+    "aws.cloudtrail.response_elements"
+]
 
 [[rule.threat]]
 framework = "MITRE ATT&CK"

rules/integrations/aws/persistence_iam_user_created_access_keys_for_another_user.toml

@@ -2,7 +2,7 @@
 creation_date = "2024/06/13"
 integration = ["aws"]
 maturity = "production"
-updated_date = "2024/10/09"
+updated_date = "2024/11/07"
 min_stack_comments = "ES|QL rule type in technical preview as of 8.13"
 min_stack_version = "8.13.0"
 
@@ -104,9 +104,39 @@ from logs-aws.cloudtrail-* metadata _id, _version, _index
     and event.action == "CreateAccessKey"
     and event.outcome == "success"
     and user.name != user.target.name
-| keep @timestamp, event.provider, event.action, event.outcome, user.name, user.target.name
+| keep
+    @timestamp,
+    cloud.region,
+    event.provider,
+    event.action,
+    event.outcome,
+    user.name,
+    source.address,
+    user.target.name,
+    user_agent.original,
+    aws.cloudtrail.request_parameters,
+    aws.cloudtrail.response_elements,
+    aws.cloudtrail.user_identity.arn,
+    aws.cloudtrail.user_identity.type,
 '''
 
+[rule.investigation_fields]
+field_names = [
+    "@timestamp",
+    "user.name",
+    "source.address",
+    "aws.cloudtrail.user_identity.arn",
+    "aws.cloudtrail.user_identity.type",
+    "user_agent.original",
+    "user.target.name",
+    "event.action",
+    "event.outcome",
+    "cloud.region",
+    "event.provider",
+    "aws.cloudtrail.request_parameters",
+    "aws.cloudtrail.response_elements",
+]
+
 
 [[rule.threat]]
 framework = "MITRE ATT&CK"

rules/integrations/aws/privilege_escalation_iam_administratoraccess_policy_attached_to_user.toml

@@ -2,7 +2,7 @@
 creation_date = "2024/05/30"
 integration = ["aws"]
 maturity = "production"
-updated_date = "2024/10/09"
+updated_date = "2024/11/07"
 min_stack_comments = "ES|QL rule type in technical preview as of 8.13."
 min_stack_version = "8.13.0"
 
@@ -103,9 +103,38 @@ from logs-aws.cloudtrail-* metadata _id, _version, _index
 | where event.provider == "iam.amazonaws.com" and event.action == "AttachUserPolicy" and event.outcome == "success"
 | dissect aws.cloudtrail.request_parameters "{%{?policyArn}=%{?arn}:%{?aws}:%{?iam}::%{?aws}:%{?policy}/%{policyName},%{?userName}=%{target.userName}}"
 | where policyName == "AdministratorAccess"
-| keep @timestamp, event.provider, event.action, event.outcome, policyName, target.userName
+| keep
+    @timestamp,
+    cloud.region,
+    event.provider,
+    event.action,
+    event.outcome,
+    policyName,
+    target.userName,
+    aws.cloudtrail.request_parameters,
+    aws.cloudtrail.user_identity.arn,
+    related.user,
+    user_agent.original,
+    user.name,
+    source.address
 '''
 
+[rule.investigation_fields]
+field_names = [
+    "@timestamp",
+    "user.name",
+    "source.address",
+    "aws.cloudtrail.user_identity.arn",
+    "user_agent.original",
+    "target.userName",
+    "event.action",
+    "policyName",
+    "event.outcome",
+    "cloud.region",
+    "event.provider",
+    "aws.cloudtrail.request_parameters"
+]
+
 
 [[rule.threat]]
 framework = "MITRE ATT&CK"

rules/integrations/aws/privilege_escalation_iam_customer_managed_policy_attached_to_role.toml

@@ -2,7 +2,7 @@
 creation_date = "2024/11/04"
 integration = ["aws"]
 maturity = "production"
-updated_date = "2024/11/04"
+updated_date = "2024/11/07"
 
 [rule]
 author = ["Elastic"]
@@ -94,9 +94,25 @@ event.dataset: "aws.cloudtrail"
     and event.provider: "iam.amazonaws.com"
     and event.action: "AttachRolePolicy"
     and event.outcome: "success"
-    and not aws.cloudtrail.flattened.request_paramters.policyArn: arn\:aws\:iam\:\:aws\:policy*
+    and not aws.cloudtrail.flattened.request_parameters.policyArn: arn\:aws\:iam\:\:aws\:policy*
 '''
 
+[rule.investigation_fields]
+field_names = [
+    "@timestamp",
+    "user.name",
+    "source.address",
+    "aws.cloudtrail.user_identity.arn",
+    "aws.cloudtrail.user_identity.type",
+    "user_agent.original",
+    "aws.cloudtrail.flattened.request_parameters.policyArn",
+    "aws.cloudtrail.flattened.request_parameters.roleName",
+    "event.action",
+    "event.outcome",
+    "cloud.region",
+    "event.provider",
+    "aws.cloudtrail.request_parameters"
+]
 
 [[rule.threat]]
 framework = "MITRE ATT&CK"

rules/integrations/aws/privilege_escalation_role_assumption_by_service.toml

@@ -2,7 +2,7 @@
 creation_date = "2021/05/17"
 integration = ["aws"]
 maturity = "production"
-updated_date = "2024/11/05"
+updated_date = "2024/11/07"
 
 [rule]
 author = ["Elastic", "Austin Songer"]
@@ -103,6 +103,22 @@ event.dataset: "aws.cloudtrail"
             )
 '''
 
+[rule.investigation_fields]
+field_names = [
+    "@timestamp",
+    "aws.cloudtrail.user_identity.type",
+    "aws.cloudtrail.resources.arn",
+    "aws.cloudtrail.resources.type",
+    "source.address",
+    "aws.cloudtrail.user_identity.invoked_by",
+    "aws.cloudtrail.flattened.request_parameters.roleArn",
+    "aws.cloudtrail.flattened.request_parameters.roleSessionName",
+    "event.action",
+    "event.outcome",
+    "cloud.region",
+    "aws.cloudtrail.request_parameters",
+    "aws.cloudtrail.response_elements"
+]
 
 [[rule.threat]]
 framework = "MITRE ATT&CK"

rules/integrations/aws/privilege_escalation_role_assumption_by_user.toml

@@ -2,7 +2,7 @@
 creation_date = "2024/11/05"
 integration = ["aws"]
 maturity = "production"
-updated_date = "2024/11/05"
+updated_date = "2024/11/07"
 
 [rule]
 author = ["Elastic"]
@@ -94,6 +94,24 @@ event.dataset: "aws.cloudtrail"
     and aws.cloudtrail.user_identity.type: ("AssumedRole" or "IAMUser")
 '''
 
+[rule.investigation_fields]
+field_names = [
+    "@timestamp",
+    "user.name",
+    "user_agent.original",
+    "source.address",
+    "aws.cloudtrail.user_identity.arn",
+    "aws.cloudtrail.user_identity.type",
+    "aws.cloudtrail.resources.arn",
+    "aws.cloudtrail.resources.type",
+    "aws.cloudtrail.flattened.request_parameters.roleArn",
+    "aws.cloudtrail.flattened.request_parameters.roleSessionName",
+    "event.action",
+    "event.outcome",
+    "cloud.region",
+    "aws.cloudtrail.request_parameters",
+    "aws.cloudtrail.response_elements"
+]
 
 [[rule.threat]]
 framework = "MITRE ATT&CK"

rules/integrations/aws_bedrock/aws_bedrock_multiple_attempts_to_use_denied_models_by_user.toml

@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2024/05/02"
 maturity = "production"
-updated_date = "2024/11/05"
+updated_date = "2024/11/07"
 min_stack_comments = "ES|QL rule type is still in technical preview as of 8.13, however this rule was tested successfully; integration in tech preview"
 min_stack_version = "8.13.0"
 
@@ -85,3 +85,11 @@ from logs-aws_bedrock.invocation-*
 | where total_denials > 3
 | sort total_denials desc
 '''
+
+[rule.investigation_fields]
+field_names = [
+    "user.id",
+    "cloud.account.id",
+    "gen_ai.request.model.id",
+    "total_denials"
+]

rules/integrations/aws_bedrock/aws_bedrock_multiple_validation_exception_errors_by_single_user.toml

@@ -2,7 +2,7 @@
 creation_date = "2024/09/11"
 integration = ["aws_bedrock"]
 maturity = "production"
-updated_date = "2024/11/05"
+updated_date = "2024/11/07"
 min_stack_comments = "ES|QL rule type is still in technical preview as of 8.13, however this rule was tested successfully; integration in tech preview"
 min_stack_version = "8.13.0"
 
@@ -92,3 +92,12 @@ from logs-aws_bedrock.invocation-*
 | stats total_denials = count(*) by target_time_window, user.id, cloud.account.id
 | where total_denials > 3
 '''
+
+[rule.investigation_fields]
+field_names = [
+    "target_time_window",
+    "user.id",
+    "cloud.account.id",
+    "total_denials"
+
+]

rules_building_block/persistence_iam_instance_request_to_iam_service.toml

@@ -3,7 +3,7 @@ bypass_bbr_timing = true
 creation_date = "2024/07/24"
 integration = ["aws"]
 maturity = "production"
-updated_date = "2024/11/05"
+updated_date = "2024/11/07"
 
 [rule]
 author = ["Elastic"]
@@ -64,6 +64,23 @@ any where event.dataset == "aws.cloudtrail"
     )
 '''
 
+[rule.investigation_fields]
+field_names = [
+    "@timestamp",
+    "source.address",
+    "user.name",
+    "user.id",
+    "aws.cloudtrail.user_identity.arn",
+    "aws.cloudtrail.user_identity.type",
+    "user_agent.original",
+    "user.target.name",
+    "event.action",
+    "event.outcome",
+    "cloud.region",
+    "event.provider",
+    "aws.cloudtrail.request_parameters",
+    "aws.cloudtrail.response_elements"
+]
 
 [[rule.threat]]
 framework = "MITRE ATT&CK"