diff --git a/detection_rules/rule.py b/detection_rules/rule.py index d57b9c86e..897c93cc7 100644 --- a/detection_rules/rule.py +++ b/detection_rules/rule.py @@ -7,6 +7,7 @@ import copy import dataclasses import json import os +import re import time import typing from abc import ABC, abstractmethod @@ -909,6 +910,22 @@ class ESQLRuleData(QueryRuleData): if data.get('index'): raise ValidationError("Index is not a valid field for ES|QL rule type.") + # Convert the query string to lowercase to handle case insensitivity + query_lower = data['query'].lower() + + # Combine both patterns using an OR operator and compile the regex + combined_pattern = re.compile( + r'(from\s+\S+\s+metadata\s+_id,\s*_version,\s*_index)|(\bstats\b.*?\bby\b)', re.DOTALL + ) + + # Ensure that non-aggregate queries have metadata + if not combined_pattern.search(query_lower): + raise ValidationError( + f"Rule: {data['name']} contains a non-aggregate query without" + f" metadata fields '_id', '_version', and '_index' ->" + f" Add 'metadata _id, _version, _index' to the from command or add an aggregate function." + ) + @dataclass(frozen=True) class ThreatMatchRuleData(QueryRuleData): diff --git a/rules/integrations/aws/exfiltration_ec2_ebs_snapshot_shared_with_another_account.toml b/rules/integrations/aws/exfiltration_ec2_ebs_snapshot_shared_with_another_account.toml index 98cac197a..926072b80 100644 --- a/rules/integrations/aws/exfiltration_ec2_ebs_snapshot_shared_with_another_account.toml +++ b/rules/integrations/aws/exfiltration_ec2_ebs_snapshot_shared_with_another_account.toml @@ -4,7 +4,7 @@ integration = ["aws"] maturity = "production" min_stack_comments = "AWS integration breaking changes, bumping version to ^2.0.0" min_stack_version = "8.13.0" -updated_date = "2024/06/02" +updated_date = "2024/10/02" [rule] author = ["Elastic"] @@ -79,7 +79,7 @@ timestamp_override = "event.ingested" type = "esql" query = ''' -from logs-aws.cloudtrail-* +from logs-aws.cloudtrail-* metadata _id, _version, _index | where event.provider == "ec2.amazonaws.com" and event.action == "ModifySnapshotAttribute" and event.outcome == "success" | dissect aws.cloudtrail.request_parameters "{%{?snapshotId}=%{snapshotId},%{?attributeType}=%{attributeType},%{?createVolumePermission}={%{operationType}={%{?items}=[{%{?userId}=%{userId}}]}}}" | where operationType == "add" and cloud.account.id != userId diff --git a/rules/integrations/aws/impact_s3_object_encryption_with_external_key.toml b/rules/integrations/aws/impact_s3_object_encryption_with_external_key.toml index 501a306fc..e5a1ff1de 100644 --- a/rules/integrations/aws/impact_s3_object_encryption_with_external_key.toml +++ b/rules/integrations/aws/impact_s3_object_encryption_with_external_key.toml @@ -4,7 +4,7 @@ integration = ["aws"] maturity = "production" min_stack_comments = "ES|QL rule type in technical preview as of 8.13" min_stack_version = "8.13.0" -updated_date = "2024/07/02" +updated_date = "2024/10/02" [rule] author = ["Elastic"] @@ -81,12 +81,12 @@ timestamp_override = "event.ingested" type = "esql" query = ''' -from logs-aws.cloudtrail-* +from logs-aws.cloudtrail-* metadata _id, _version, _index // any successful copy event -| where event.dataset == "aws.cloudtrail" - and event.provider == "s3.amazonaws.com" - and event.action == "CopyObject" +| where event.dataset == "aws.cloudtrail" + and event.provider == "s3.amazonaws.com" + and event.action == "CopyObject" and event.outcome == "success" // abstract key account id, key id, encrypted object bucket name and object name @@ -94,7 +94,7 @@ from logs-aws.cloudtrail-* // filter for s3 objects whose account id is different from the encryption key's account id // add exceptions based on key.account.id or keyId for known external accounts or encryption keys -| where cloud.account.id != key.account.id +| where cloud.account.id != key.account.id ''' diff --git a/rules/integrations/aws/initial_access_signin_console_login_no_mfa.toml b/rules/integrations/aws/initial_access_signin_console_login_no_mfa.toml index 5cb8fbe0d..e64319493 100644 --- a/rules/integrations/aws/initial_access_signin_console_login_no_mfa.toml +++ b/rules/integrations/aws/initial_access_signin_console_login_no_mfa.toml @@ -2,7 +2,7 @@ creation_date = "2024/08/19" integration = ['aws'] maturity = "production" -updated_date = "2024/08/19" +updated_date = "2024/10/02" min_stack_comments = "ES|QL rule type in technical preview as of 8.13" min_stack_version = "8.13.0" @@ -36,7 +36,7 @@ timestamp_override = "event.ingested" type = "esql" query = ''' -from logs-aws.cloudtrail-* +from logs-aws.cloudtrail-* metadata _id, _version, _index | where event.provider == "signin.amazonaws.com" and event.action == "GetSigninToken" diff --git a/rules/integrations/aws/persistence_iam_user_created_access_keys_for_another_user.toml b/rules/integrations/aws/persistence_iam_user_created_access_keys_for_another_user.toml index 91018b139..830eb8a73 100644 --- a/rules/integrations/aws/persistence_iam_user_created_access_keys_for_another_user.toml +++ b/rules/integrations/aws/persistence_iam_user_created_access_keys_for_another_user.toml @@ -2,7 +2,7 @@ creation_date = "2024/06/13" integration = ["aws"] maturity = "production" -updated_date = "2024/07/06" +updated_date = "2024/10/02" min_stack_comments = "ES|QL rule type in technical preview as of 8.13" min_stack_version = "8.13.0" @@ -11,12 +11,12 @@ author = ["Elastic"] description = """ An adversary with access to a set of compromised credentials may attempt to persist or escalate privileges by creating a new set of credentials for an existing user. This rule looks for use of the IAM `CreateAccessKey` API operation -to create new programatic access keys for another IAM user. +to create new programmatic access keys for another IAM user. """ false_positives = [ """ While this can be normal behavior, it should be investigated to ensure validity. - Verify whether the user identity should be using the IAM `CreateAccessKey` for the targeted user. + Verify whether the user identity should be using the IAM `CreateAccessKey` for the targeted user. """, ] from = "now-6m" @@ -27,8 +27,8 @@ note = """## Triage and analysis ### Investigating AWS IAM User Created Access Keys For Another User -AWS access keys created for IAM users or root user are long-term credentials that provide programatic access to AWS. -With access to the `iam:CreateAccessKey` permission, a set of compromised credentials could be used to create a new +AWS access keys created for IAM users or root user are long-term credentials that provide programmatic access to AWS. +With access to the `iam:CreateAccessKey` permission, a set of compromised credentials could be used to create a new set of credentials for another user for privilege escalation or as a means of persistence. This rule uses [ES|QL](https://www.elastic.co/guide/en/security/master/rules-ui-create.html#create-esql-rule) to look for use of the `CreateAccessKey` operation where the user.name is different from the user.target.name. @@ -65,8 +65,8 @@ to look for use of the `CreateAccessKey` operation where the user.name is differ - Work with your IT team to identify and minimize the impact on users. - Identify if the attacker is moving laterally and compromising other accounts, servers, or services. - Identify any regulatory or legal ramifications related to this activity. -- Investigate credential exposure on systems compromised or used by the attacker to ensure all compromised accounts are identified. - - Rotate secrets or delete API keys as needed to revoke the attacker's access to the environment. +- Investigate credential exposure on systems compromised or used by the attacker to ensure all compromised accounts are identified. + - Rotate secrets or delete API keys as needed to revoke the attacker's access to the environment. - Work with your IT teams to minimize the impact on business operations during these actions. - Remove unauthorized new accounts, and request password resets for other IAM users. - Consider enabling multi-factor authentication for users. @@ -99,8 +99,11 @@ timestamp_override = "event.ingested" type = "esql" query = ''' -from logs-aws.cloudtrail-* -| where event.provider == "iam.amazonaws.com" and event.action == "CreateAccessKey" and event.outcome == "success" and user.name != user.target.name +from logs-aws.cloudtrail-* metadata _id, _version, _index +| where event.provider == "iam.amazonaws.com" + and event.action == "CreateAccessKey" + and event.outcome == "success" + and user.name != user.target.name ''' diff --git a/rules/integrations/aws/privilege_escalation_iam_administratoraccess_policy_attached_to_group.toml b/rules/integrations/aws/privilege_escalation_iam_administratoraccess_policy_attached_to_group.toml index 2282733cf..1b23250ba 100644 --- a/rules/integrations/aws/privilege_escalation_iam_administratoraccess_policy_attached_to_group.toml +++ b/rules/integrations/aws/privilege_escalation_iam_administratoraccess_policy_attached_to_group.toml @@ -2,7 +2,7 @@ creation_date = "2024/05/31" integration = ["aws"] maturity = "production" -updated_date = "2024/07/05" +updated_date = "2024/10/02" min_stack_comments = "ES|QL rule type in technical preview as of 8.13." min_stack_version = "8.13.0" @@ -10,15 +10,15 @@ min_stack_version = "8.13.0" author = ["Elastic"] description = """ An adversary with access to a set of compromised credentials may attempt to persist or escalate privileges by -attaching additional permissions to user groups the compromised user account belongs to. This rule looks for use of -the IAM `AttachGroupPolicy` API operation to attach the highly permissive `AdministratorAccess` AWS managed policy +attaching additional permissions to user groups the compromised user account belongs to. This rule looks for use of +the IAM `AttachGroupPolicy` API operation to attach the highly permissive `AdministratorAccess` AWS managed policy to an existing IAM user group. """ false_positives = [ """ While this can be normal behavior, it should be investigated to ensure validity. Verify whether the user identity should be using the IAM `AttachGroupPolicy` API operation - to attach the `AdministratorAccess` policy to the user group. + to attach the `AdministratorAccess` policy to the user group. """, ] from = "now-6m" @@ -29,7 +29,7 @@ note = """## Triage and analysis ### Investigating AWS IAM AdministratorAccess Policy Attached to Group -The AWS IAM `AdministratorAccess` managed policy provides full access to all AWS services and resources. +The AWS IAM `AdministratorAccess` managed policy provides full access to all AWS services and resources. With access to the `iam:AttachGroupPolicy` permission, a set of compromised credentials could be used to attach this policy to the current user's groups for privilege escalation or as a means of persistence. This rule uses [ES|QL](https://www.elastic.co/guide/en/security/master/rules-ui-create.html#create-esql-rule) to look for use of the `AttachGroupPolicy` operation along with request_parameters where the policyName is `AdministratorAccess`. @@ -67,8 +67,8 @@ to look for use of the `AttachGroupPolicy` operation along with request_paramete - Work with your IT team to identify and minimize the impact on users. - Identify if the attacker is moving laterally and compromising other accounts, servers, or services. - Identify any regulatory or legal ramifications related to this activity. -- Investigate credential exposure on systems compromised or used by the attacker to ensure all compromised accounts are identified. - - Rotate secrets or delete API keys as needed to revoke the attacker's access to the environment. +- Investigate credential exposure on systems compromised or used by the attacker to ensure all compromised accounts are identified. + - Rotate secrets or delete API keys as needed to revoke the attacker's access to the environment. - Work with your IT teams to minimize the impact on business operations during these actions. - Check if unauthorized new users were created, remove unauthorized new accounts, and request password resets for other IAM users. - Consider enabling multi-factor authentication for users. @@ -100,7 +100,7 @@ timestamp_override = "event.ingested" type = "esql" query = ''' -from logs-aws.cloudtrail-* +from logs-aws.cloudtrail-* metadata _id, _version, _index | where event.provider == "iam.amazonaws.com" and event.action == "AttachGroupPolicy" and event.outcome == "success" | dissect aws.cloudtrail.request_parameters "{%{?policyArn}=%{?arn}:%{?aws}:%{?iam}::%{?aws}:%{?policy}/%{policyName},%{?groupName}=%{group.name}}" | where policyName == "AdministratorAccess" diff --git a/rules/integrations/aws/privilege_escalation_iam_administratoraccess_policy_attached_to_role.toml b/rules/integrations/aws/privilege_escalation_iam_administratoraccess_policy_attached_to_role.toml index 315b25f19..aa70d2575 100644 --- a/rules/integrations/aws/privilege_escalation_iam_administratoraccess_policy_attached_to_role.toml +++ b/rules/integrations/aws/privilege_escalation_iam_administratoraccess_policy_attached_to_role.toml @@ -2,7 +2,7 @@ creation_date = "2024/05/31" integration = ["aws"] maturity = "production" -updated_date = "2024/07/05" +updated_date = "2024/10/02" min_stack_comments = "ES|QL rule type in technical preview as of 8.13." min_stack_version = "8.13.0" @@ -17,7 +17,7 @@ false_positives = [ """ While this can be normal behavior, it should be investigated to ensure validity. Verify whether the user identity should be using the IAM `AttachRolePolicy` API operation - to attach the `AdministratorAccess` policy to the target role. + to attach the `AdministratorAccess` policy to the target role. """, ] from = "now-6m" @@ -28,7 +28,7 @@ note = """## Triage and analysis ### Investigating AWS IAM AdministratorAccess Policy Attached to Role -The AWS IAM `AdministratorAccess` managed policy provides full access to all AWS services and resources. +The AWS IAM `AdministratorAccess` managed policy provides full access to all AWS services and resources. With access to the `iam:AttachRolePolicy` permission, a set of compromised credentials could be used to attach this policy to a compromised role for privilege escalation or as a means of persistence. This rule uses [ES|QL](https://www.elastic.co/guide/en/security/master/rules-ui-create.html#create-esql-rule) to look for use of the `AttachRolePolicy` operation along with request_parameters where the policyName is `AdministratorAccess`. @@ -66,8 +66,8 @@ to look for use of the `AttachRolePolicy` operation along with request_parameter - Work with your IT team to identify and minimize the impact on users. - Identify if the attacker is moving laterally and compromising other accounts, servers, or services. - Identify any regulatory or legal ramifications related to this activity. -- Investigate credential exposure on systems compromised or used by the attacker to ensure all compromised accounts are identified. - - Rotate secrets or delete API keys as needed to revoke the attacker's access to the environment. +- Investigate credential exposure on systems compromised or used by the attacker to ensure all compromised accounts are identified. + - Rotate secrets or delete API keys as needed to revoke the attacker's access to the environment. - Work with your IT teams to minimize the impact on business operations during these actions. - Check if unauthorized new users were created, remove unauthorized new accounts, and request password resets for other IAM users. - Consider enabling multi-factor authentication for users. @@ -99,7 +99,7 @@ timestamp_override = "event.ingested" type = "esql" query = ''' -from logs-aws.cloudtrail-* +from logs-aws.cloudtrail-* metadata _id, _version, _index | where event.provider == "iam.amazonaws.com" and event.action == "AttachRolePolicy" and event.outcome == "success" | dissect aws.cloudtrail.request_parameters "{%{?policyArn}=%{?arn}:%{?aws}:%{?iam}::%{?aws}:%{?policy}/%{policyName},%{?roleName}=%{role.name}}" | where policyName == "AdministratorAccess" diff --git a/rules/integrations/aws/privilege_escalation_iam_administratoraccess_policy_attached_to_user.toml b/rules/integrations/aws/privilege_escalation_iam_administratoraccess_policy_attached_to_user.toml index f1702a82a..fa81755a3 100644 --- a/rules/integrations/aws/privilege_escalation_iam_administratoraccess_policy_attached_to_user.toml +++ b/rules/integrations/aws/privilege_escalation_iam_administratoraccess_policy_attached_to_user.toml @@ -2,7 +2,7 @@ creation_date = "2024/05/30" integration = ["aws"] maturity = "production" -updated_date = "2024/07/05" +updated_date = "2024/10/02" min_stack_comments = "ES|QL rule type in technical preview as of 8.13." min_stack_version = "8.13.0" @@ -17,7 +17,7 @@ false_positives = [ """ While this can be normal behavior, it should be investigated to ensure validity. Verify whether the user identity should be using the IAM `AttachUserPolicy` API operation - to attach the `AdministratorAccess` policy to the target user. + to attach the `AdministratorAccess` policy to the target user. """, ] from = "now-6m" @@ -28,7 +28,7 @@ note = """## Triage and analysis ### Investigating AWS IAM AdministratorAccess Policy Attached to User -The AWS IAM `AdministratorAccess` managed policy provides full access to all AWS services and resources. +The AWS IAM `AdministratorAccess` managed policy provides full access to all AWS services and resources. With access to the `iam:AttachUserPolicy` permission, a set of compromised credentials could be used to attach this policy to the current user for privilege escalation or another user as a means of persistence. This rule uses [ES|QL](https://www.elastic.co/guide/en/security/master/rules-ui-create.html#create-esql-rule) to look for use of the `AttachUserPolicy` operation along with request_parameters where the policyName is `AdministratorAccess`. @@ -66,8 +66,8 @@ to look for use of the `AttachUserPolicy` operation along with request_parameter - Work with your IT team to identify and minimize the impact on users. - Identify if the attacker is moving laterally and compromising other accounts, servers, or services. - Identify any regulatory or legal ramifications related to this activity. -- Investigate credential exposure on systems compromised or used by the attacker to ensure all compromised accounts are identified. - - Rotate secrets or delete API keys as needed to revoke the attacker's access to the environment. +- Investigate credential exposure on systems compromised or used by the attacker to ensure all compromised accounts are identified. + - Rotate secrets or delete API keys as needed to revoke the attacker's access to the environment. - Work with your IT teams to minimize the impact on business operations during these actions. - Check if unauthorized new users were created, remove unauthorized new accounts, and request password resets for other IAM users. - Consider enabling multi-factor authentication for users. @@ -99,7 +99,7 @@ timestamp_override = "event.ingested" type = "esql" query = ''' -from logs-aws.cloudtrail-* +from logs-aws.cloudtrail-* metadata _id, _version, _index | where event.provider == "iam.amazonaws.com" and event.action == "AttachUserPolicy" and event.outcome == "success" | dissect aws.cloudtrail.request_parameters "{%{?policyArn}=%{?arn}:%{?aws}:%{?iam}::%{?aws}:%{?policy}/%{policyName},%{?userName}=%{target.userName}}" | where policyName == "AdministratorAccess"