diff --git a/detection_rules/devtools.py b/detection_rules/devtools.py index 5dfbf37af..f32d6af8c 100644 --- a/detection_rules/devtools.py +++ b/detection_rules/devtools.py @@ -1236,14 +1236,19 @@ def build_integration_manifests(overwrite: bool, integration: str): @integrations_group.command('build-schemas') @click.option('--overwrite', '-o', is_flag=True, help="Overwrite the entire integrations-schema.json.gz file") -def build_integration_schemas(overwrite: bool): +@click.option('--integration', '-i', type=str, + help="Adds a single integration schema to the integrations-schema.json.gz file") +def build_integration_schemas(overwrite: bool, integration: str): """Builds consolidated integrations schemas file.""" click.echo("Building integration schemas...") start_time = time.perf_counter() - build_integrations_schemas(overwrite) - end_time = time.perf_counter() - click.echo(f"Time taken to generate schemas: {(end_time - start_time)/60:.2f} minutes") + if integration: + build_integrations_schemas(overwrite=False, integration=integration) + else: + build_integrations_schemas(overwrite=overwrite) + end_time = time.perf_counter() + click.echo(f"Time taken to generate schemas: {(end_time - start_time)/60:.2f} minutes") @integrations_group.command('show-latest-compatible') diff --git a/detection_rules/etc/integration-manifests.json.gz b/detection_rules/etc/integration-manifests.json.gz index 5303e11f2..897bd5e1a 100644 Binary files a/detection_rules/etc/integration-manifests.json.gz and b/detection_rules/etc/integration-manifests.json.gz differ diff --git a/detection_rules/etc/integration-schemas.json.gz b/detection_rules/etc/integration-schemas.json.gz index dc8f49bd6..930510ffc 100644 Binary files a/detection_rules/etc/integration-schemas.json.gz and b/detection_rules/etc/integration-schemas.json.gz differ diff --git a/detection_rules/integrations.py b/detection_rules/integrations.py index 16f6999a2..faa305175 100644 --- a/detection_rules/integrations.py +++ b/detection_rules/integrations.py @@ -47,12 +47,13 @@ class IntegrationManifestSchema(Schema): description = fields.Str(required=True) download = fields.Str(required=True) conditions = fields.Dict(required=True) - policy_templates = fields.List(fields.Dict, required=True) + policy_templates = fields.List(fields.Dict) owner = fields.Dict(required=False) @post_load def transform_policy_template(self, data, **kwargs): - data["policy_templates"] = [policy["name"] for policy in data["policy_templates"]] + if "policy_templates" in data: + data["policy_templates"] = [policy["name"] for policy in data["policy_templates"]] return data @@ -93,21 +94,30 @@ def build_integrations_manifest(overwrite: bool, rule_integrations: list = [], i print(f"final integrations manifests dumped: {MANIFEST_FILE_PATH}") -def build_integrations_schemas(overwrite: bool) -> None: +def build_integrations_schemas(overwrite: bool, integration: str = None) -> None: """Builds a new local copy of integration-schemas.json.gz from EPR integrations.""" - final_integration_schemas = {} saved_integration_schemas = {} # Check if the file already exists and handle accordingly if overwrite and SCHEMA_FILE_PATH.exists(): SCHEMA_FILE_PATH.unlink() + final_integration_schemas = {} elif SCHEMA_FILE_PATH.exists(): - saved_integration_schemas = load_integrations_schemas() + final_integration_schemas = load_integrations_schemas() + else: + final_integration_schemas = {} # Load the integration manifests integration_manifests = load_integrations_manifests() + # if a single integration is specified, only process that integration + if integration: + if integration in integration_manifests: + integration_manifests = {integration: integration_manifests[integration]} + else: + raise ValueError(f"Integration {integration} not found in manifest.") + # Loop through the packages and versions for package, versions in integration_manifests.items(): print(f"processing {package}") diff --git a/detection_rules/rule.py b/detection_rules/rule.py index 61cbf9a20..b7679b9ad 100644 --- a/detection_rules/rule.py +++ b/detection_rules/rule.py @@ -1024,8 +1024,10 @@ class TOMLRuleContents(BaseRuleContents, MarshmallowDataclassMixin): # if integration is not a policy template remove if package["version"]: - policy_templates = packages_manifest[ - package["package"]][package["version"].strip("^")]["policy_templates"] + version_data = packages_manifest.get(package["package"], + {}).get(package["version"].strip("^"), {}) + policy_templates = version_data.get("policy_templates", []) + if package["integration"] not in policy_templates: del package["integration"] @@ -1131,7 +1133,9 @@ class TOMLRuleContents(BaseRuleContents, MarshmallowDataclassMixin): rule_integrations = meta.get("integration", []) if rule_integrations: for integration in rule_integrations: - if integration in definitions.NON_DATASET_PACKAGES or isinstance(data, MachineLearningRuleData): + ineligible_integrations = definitions.NON_DATASET_PACKAGES + \ + [*map(str.lower, definitions.MACHINE_LEARNING_PACKAGES)] + if integration in ineligible_integrations or isinstance(data, MachineLearningRuleData): packaged_integrations.append({"package": integration, "integration": None}) for value in sorted(datasets): diff --git a/detection_rules/schemas/definitions.py b/detection_rules/schemas/definitions.py index cb842b5e1..d43d36aa6 100644 --- a/detection_rules/schemas/definitions.py +++ b/detection_rules/schemas/definitions.py @@ -125,6 +125,7 @@ EXPECTED_RULE_TAGS = [ 'Use Case: Vulnerability' ] +MACHINE_LEARNING_PACKAGES = ['LMD', 'DGA', 'DED', 'ProblemChild', 'Beaconing'] NonEmptyStr = NewType('NonEmptyStr', str, validate=validate.Length(min=1)) TimeUnits = Literal['s', 'm', 'h'] @@ -159,5 +160,6 @@ UUIDString = NewType('UUIDString', str, validate=validate.Regexp(UUID_PATTERN)) BuildingBlockType = Literal['default'] # experimental machine learning features and releases -MachineLearningType = Literal['DGA', 'ProblemChild'] -MachineLearningTypeLower = Literal['dga', 'problemchild'] +MachineLearningType = getattr(Literal, '__getitem__')(tuple(MACHINE_LEARNING_PACKAGES)) # noqa: E999 +MachineLearningTypeLower = getattr(Literal, '__getitem__')( + tuple(map(str.lower, MACHINE_LEARNING_PACKAGES))) # noqa: E999 diff --git a/rules/integrations/lmd/lateral_movement_malicious_remote_file_creation.toml b/rules/integrations/lmd/lateral_movement_malicious_remote_file_creation.toml new file mode 100644 index 000000000..19ea2b676 --- /dev/null +++ b/rules/integrations/lmd/lateral_movement_malicious_remote_file_creation.toml @@ -0,0 +1,45 @@ +[metadata] +creation_date = "2023/09/13" +integration = ["lmd","endpoint"] +maturity = "production" +min_stack_comments = "New fields added: required_fields, related_integrations, setup" +min_stack_version = "8.3.0" +updated_date = "2023/09/21" + +[rule] +author = ["Elastic"] +description = "Malicious remote file creation, which can be an indicator of lateral movement activity." +from = "now-10m" +index = ["logs-endpoint.events.*"] +interval = "5m" +language = "eql" +license = "Elastic License v2" +name = "Malicious Remote File Creation" +references = ["https://www.elastic.co/es/blog/remote-desktop-protocol-connections-elastic-security"] +risk_score = 99 +rule_id = "301571f3-b316-4969-8dd0-7917410030d3" +severity = "critical" +tags = ["Domain: Endpoint", "Use Case: Lateral Movement Detection", "Tactic: Lateral Movement", "Data Source: Elastic Defend"] +type = "eql" + +query = ''' +sequence by host.name +[file where event.action == "creation" and process.name : ("System", "scp", "sshd", "smbd", "vsftpd", "sftp-server")] +[file where event.category == "malware" or event.category == "intrusion_detection" +and process.name:("System", "scp", "sshd", "smbd", "vsftpd", "sftp-server")] +''' + + +[[rule.threat]] +framework = "MITRE ATT&CK" +[[rule.threat.technique]] +id = "T1210" +name = "Exploitation of Remote Services" +reference = "https://attack.mitre.org/techniques/T1210/" + + +[rule.threat.tactic] +id = "TA0008" +name = "Lateral Movement" +reference = "https://attack.mitre.org/tactics/TA0008/" + diff --git a/rules/integrations/lmd/lateral_movement_ml_high_mean_rdp_process_args.toml b/rules/integrations/lmd/lateral_movement_ml_high_mean_rdp_process_args.toml new file mode 100644 index 000000000..1258f9bba --- /dev/null +++ b/rules/integrations/lmd/lateral_movement_ml_high_mean_rdp_process_args.toml @@ -0,0 +1,45 @@ +[metadata] +creation_date = "2023/09/13" +integration = ["lmd"] +maturity = "production" +min_stack_comments = "New fields added: required_fields, related_integrations, setup" +min_stack_version = "8.3.0" +updated_date = "2023/09/21" + +[rule] +anomaly_threshold = 70 +author = ["Elastic"] +description = """ +A machine learning job has detected unusually high number of process arguments in an RDP session. Executing +sophisticated attacks such as lateral movement can involve the use of complex commands, obfuscation mechanisms, +redirection and piping, which in turn increases the number of arguments in a command. +""" +from = "now-12h" +interval = "15m" +license = "Elastic License v2" +machine_learning_job_id = "lmd_high_mean_rdp_process_args" +name = "High Mean of Process Arguments in an RDP Session" +references = ["https://www.elastic.co/guide/en/security/current/prebuilt-ml-jobs.html"] +risk_score = 21 +rule_id = "36c48a0c-c63a-4cbc-aee1-8cac87db31a9" +severity = "low" +tags = [ + "Use Case: Lateral Movement Detection", + "Rule Type: ML", + "Rule Type: Machine Learning", + "Tactic: Lateral Movement", +] +type = "machine_learning" +[[rule.threat]] +framework = "MITRE ATT&CK" +[[rule.threat.technique]] +id = "T1210" +name = "Exploitation of Remote Services" +reference = "https://attack.mitre.org/techniques/T1210/" + + +[rule.threat.tactic] +id = "TA0008" +name = "Lateral Movement" +reference = "https://attack.mitre.org/tactics/TA0008/" + diff --git a/rules/integrations/lmd/lateral_movement_ml_high_mean_rdp_session_duration.toml b/rules/integrations/lmd/lateral_movement_ml_high_mean_rdp_session_duration.toml new file mode 100644 index 000000000..6a3e1989b --- /dev/null +++ b/rules/integrations/lmd/lateral_movement_ml_high_mean_rdp_session_duration.toml @@ -0,0 +1,45 @@ +[metadata] +creation_date = "2023/09/12" +integration = ["lmd"] +maturity = "production" +min_stack_comments = "New fields added: required_fields, related_integrations, setup" +min_stack_version = "8.3.0" +updated_date = "2023/09/21" + +[rule] +anomaly_threshold = 70 +author = ["Elastic"] +description = """ +A machine learning job has detected unusually high mean of RDP session duration. Long RDP sessions can be used to evade +detection mechanisms via session persistence, and might be used to perform tasks such as lateral movement, that might +require uninterrupted access to a compromised machine. +""" +from = "now-12h" +interval = "15m" +license = "Elastic License v2" +machine_learning_job_id = "lmd_high_mean_rdp_session_duration" +name = "High Mean of RDP Session Duration" +references = ["https://www.elastic.co/guide/en/security/current/prebuilt-ml-jobs.html"] +risk_score = 21 +rule_id = "a74c60cb-70ee-4629-a127-608ead14ebf1" +severity = "low" +tags = [ + "Use Case: Lateral Movement Detection", + "Rule Type: ML", + "Rule Type: Machine Learning", + "Tactic: Lateral Movement", +] +type = "machine_learning" +[[rule.threat]] +framework = "MITRE ATT&CK" +[[rule.threat.technique]] +id = "T1210" +name = "Exploitation of Remote Services" +reference = "https://attack.mitre.org/techniques/T1210/" + + +[rule.threat.tactic] +id = "TA0008" +name = "Lateral Movement" +reference = "https://attack.mitre.org/tactics/TA0008/" + diff --git a/rules/integrations/lmd/lateral_movement_ml_high_remote_file_size.toml b/rules/integrations/lmd/lateral_movement_ml_high_remote_file_size.toml new file mode 100644 index 000000000..2861af3bd --- /dev/null +++ b/rules/integrations/lmd/lateral_movement_ml_high_remote_file_size.toml @@ -0,0 +1,46 @@ +[metadata] +creation_date = "2023/09/13" +integration = ["lmd"] +maturity = "production" +min_stack_comments = "New fields added: required_fields, related_integrations, setup" +min_stack_version = "8.3.0" +updated_date = "2023/09/21" + +[rule] +anomaly_threshold = 70 +author = ["Elastic"] +description = """ +A machine learning job has detected an unusually high file size shared by a remote host indicating potential lateral +movement activity. One of the primary goals of attackers after gaining access to a network is to locate and exfiltrate +valuable information. Instead of multiple small transfers that can raise alarms, attackers might choose to bundle data +into a single large file transfer. +""" +from = "now-90m" +interval = "15m" +license = "Elastic License v2" +machine_learning_job_id = "lmd_high_file_size_remote_file_transfer" +name = "Unusual Remote File Size" +references = ["https://www.elastic.co/guide/en/security/current/prebuilt-ml-jobs.html"] +risk_score = 21 +rule_id = "0678bc9c-b71a-433b-87e6-2f664b6b3131" +severity = "low" +tags = [ + "Use Case: Lateral Movement Detection", + "Rule Type: ML", + "Rule Type: Machine Learning", + "Tactic: Lateral Movement", +] +type = "machine_learning" +[[rule.threat]] +framework = "MITRE ATT&CK" +[[rule.threat.technique]] +id = "T1210" +name = "Exploitation of Remote Services" +reference = "https://attack.mitre.org/techniques/T1210/" + + +[rule.threat.tactic] +id = "TA0008" +name = "Lateral Movement" +reference = "https://attack.mitre.org/tactics/TA0008/" + diff --git a/rules/integrations/lmd/lateral_movement_ml_high_variance_rdp_session_duration.toml b/rules/integrations/lmd/lateral_movement_ml_high_variance_rdp_session_duration.toml new file mode 100644 index 000000000..55a84e1d7 --- /dev/null +++ b/rules/integrations/lmd/lateral_movement_ml_high_variance_rdp_session_duration.toml @@ -0,0 +1,45 @@ +[metadata] +creation_date = "2023/09/13" +integration = ["lmd"] +maturity = "production" +min_stack_comments = "New fields added: required_fields, related_integrations, setup" +min_stack_version = "8.3.0" +updated_date = "2023/09/21" + +[rule] +anomaly_threshold = 70 +author = ["Elastic"] +description = """ +A machine learning job has detected unusually high variance of RDP session duration. Long RDP sessions can be used to +evade detection mechanisms via session persistence, and might be used to perform tasks such as lateral movement, that +might require uninterrupted access to a compromised machine. +""" +from = "now-12h" +interval = "15m" +license = "Elastic License v2" +machine_learning_job_id = "lmd_high_var_rdp_session_duration" +name = "High Variance in RDP Session Duration" +references = ["https://www.elastic.co/guide/en/security/current/prebuilt-ml-jobs.html"] +risk_score = 21 +rule_id = "a8d35ca0-ad8d-48a9-9f6c-553622dca61a" +severity = "low" +tags = [ + "Use Case: Lateral Movement Detection", + "Rule Type: ML", + "Rule Type: Machine Learning", + "Tactic: Lateral Movement", +] +type = "machine_learning" +[[rule.threat]] +framework = "MITRE ATT&CK" +[[rule.threat.technique]] +id = "T1210" +name = "Exploitation of Remote Services" +reference = "https://attack.mitre.org/techniques/T1210/" + + +[rule.threat.tactic] +id = "TA0008" +name = "Lateral Movement" +reference = "https://attack.mitre.org/tactics/TA0008/" + diff --git a/rules/integrations/lmd/lateral_movement_ml_rare_remote_file_directory.toml b/rules/integrations/lmd/lateral_movement_ml_rare_remote_file_directory.toml new file mode 100644 index 000000000..27da10305 --- /dev/null +++ b/rules/integrations/lmd/lateral_movement_ml_rare_remote_file_directory.toml @@ -0,0 +1,45 @@ +[metadata] +creation_date = "2023/09/12" +integration = ["lmd"] +maturity = "production" +min_stack_comments = "New fields added: required_fields, related_integrations, setup" +min_stack_version = "8.3.0" +updated_date = "2023/09/21" + +[rule] +anomaly_threshold = 70 +author = ["Elastic"] +description = """ +An anomaly detection job has detected a remote file transfer on an unusual directory indicating a potential lateral +movement activity on the host. Many Security solutions monitor well-known directories for suspicious activities, so +attackers might use less common directories to bypass monitoring. +""" +from = "now-90m" +interval = "15m" +license = "Elastic License v2" +machine_learning_job_id = "lmd_rare_file_path_remote_transfer" +name = "Unusual Remote File Directory" +references = ["https://www.elastic.co/guide/en/security/current/prebuilt-ml-jobs.html"] +risk_score = 21 +rule_id = "be4c5aed-90f5-4221-8bd5-7ab3a4334751" +severity = "low" +tags = [ + "Use Case: Lateral Movement Detection", + "Rule Type: ML", + "Rule Type: Machine Learning", + "Tactic: Lateral Movement", +] +type = "machine_learning" +[[rule.threat]] +framework = "MITRE ATT&CK" +[[rule.threat.technique]] +id = "T1210" +name = "Exploitation of Remote Services" +reference = "https://attack.mitre.org/techniques/T1210/" + + +[rule.threat.tactic] +id = "TA0008" +name = "Lateral Movement" +reference = "https://attack.mitre.org/tactics/TA0008/" + diff --git a/rules/integrations/lmd/lateral_movement_ml_rare_remote_file_extension.toml b/rules/integrations/lmd/lateral_movement_ml_rare_remote_file_extension.toml new file mode 100644 index 000000000..6e6949a6c --- /dev/null +++ b/rules/integrations/lmd/lateral_movement_ml_rare_remote_file_extension.toml @@ -0,0 +1,44 @@ +[metadata] +creation_date = "2023/09/13" +integration = ["lmd"] +maturity = "production" +min_stack_comments = "New fields added: required_fields, related_integrations, setup" +min_stack_version = "8.3.0" +updated_date = "2023/09/21" + +[rule] +anomaly_threshold = 70 +author = ["Elastic"] +description = """ +An anomaly detection job has detected a remote file transfer with a rare extension, which could indicate potential +lateral movement activity on the host. +""" +from = "now-90m" +interval = "15m" +license = "Elastic License v2" +machine_learning_job_id = "lmd_rare_file_extension_remote_transfer" +name = "Unusual Remote File Extension" +references = ["https://www.elastic.co/guide/en/security/current/prebuilt-ml-jobs.html"] +risk_score = 21 +rule_id = "814d96c7-2068-42aa-ba8e-fe0ddd565e2e" +severity = "low" +tags = [ + "Use Case: Lateral Movement Detection", + "Rule Type: ML", + "Rule Type: Machine Learning", + "Tactic: Lateral Movement", +] +type = "machine_learning" +[[rule.threat]] +framework = "MITRE ATT&CK" +[[rule.threat.technique]] +id = "T1210" +name = "Exploitation of Remote Services" +reference = "https://attack.mitre.org/techniques/T1210/" + + +[rule.threat.tactic] +id = "TA0008" +name = "Lateral Movement" +reference = "https://attack.mitre.org/tactics/TA0008/" + diff --git a/rules/integrations/lmd/lateral_movement_ml_spike_in_connections_from_a_source_ip.toml b/rules/integrations/lmd/lateral_movement_ml_spike_in_connections_from_a_source_ip.toml new file mode 100644 index 000000000..94fbb3d40 --- /dev/null +++ b/rules/integrations/lmd/lateral_movement_ml_spike_in_connections_from_a_source_ip.toml @@ -0,0 +1,45 @@ +[metadata] +creation_date = "2023/09/13" +integration = ["lmd"] +maturity = "production" +min_stack_comments = "New fields added: required_fields, related_integrations, setup" +min_stack_version = "8.3.0" +updated_date = "2023/09/21" + +[rule] +anomaly_threshold = 70 +author = ["Elastic"] +description = """ +A machine learning job has detected a high count of destination IPs establishing an RDP connection with a single source +IP. Once an attacker has gained access to one system, they might attempt to access more in the network in search of +valuable assets, data, or further access points. +""" +from = "now-12h" +interval = "15m" +license = "Elastic License v2" +machine_learning_job_id = "lmd_high_rdp_distinct_count_destination_ip_for_source" +name = "Spike in Number of Connections Made from a Source IP" +references = ["https://www.elastic.co/guide/en/security/current/prebuilt-ml-jobs.html"] +risk_score = 21 +rule_id = "3e0561b5-3fac-4461-84cc-19163b9aaa61" +severity = "low" +tags = [ + "Use Case: Lateral Movement Detection", + "Rule Type: ML", + "Rule Type: Machine Learning", + "Tactic: Lateral Movement", +] +type = "machine_learning" +[[rule.threat]] +framework = "MITRE ATT&CK" +[[rule.threat.technique]] +id = "T1210" +name = "Exploitation of Remote Services" +reference = "https://attack.mitre.org/techniques/T1210/" + + +[rule.threat.tactic] +id = "TA0008" +name = "Lateral Movement" +reference = "https://attack.mitre.org/tactics/TA0008/" + diff --git a/rules/integrations/lmd/lateral_movement_ml_spike_in_connections_to_a_destination_ip.toml b/rules/integrations/lmd/lateral_movement_ml_spike_in_connections_to_a_destination_ip.toml new file mode 100644 index 000000000..e0cd42f3e --- /dev/null +++ b/rules/integrations/lmd/lateral_movement_ml_spike_in_connections_to_a_destination_ip.toml @@ -0,0 +1,45 @@ +[metadata] +creation_date = "2023/09/13" +integration = ["lmd"] +maturity = "production" +min_stack_comments = "New fields added: required_fields, related_integrations, setup" +min_stack_version = "8.3.0" +updated_date = "2023/09/21" + +[rule] +anomaly_threshold = 70 +author = ["Elastic"] +description = """ +A machine learning job has detected a high count of source IPs establishing an RDP connection with a single destination +IP. Attackers might use multiple compromised systems to attack a target to ensure redundancy in case a source IP gets +detected and blocked. +""" +from = "now-12h" +interval = "15m" +license = "Elastic License v2" +machine_learning_job_id = "lmd_high_rdp_distinct_count_source_ip_for_destination" +name = "Spike in Number of Connections Made to a Destination IP" +references = ["https://www.elastic.co/guide/en/security/current/prebuilt-ml-jobs.html"] +risk_score = 21 +rule_id = "18a5dd9a-e3fa-4996-99b1-ae533b8f27fc" +severity = "low" +tags = [ + "Use Case: Lateral Movement Detection", + "Rule Type: ML", + "Rule Type: Machine Learning", + "Tactic: Lateral Movement", +] +type = "machine_learning" +[[rule.threat]] +framework = "MITRE ATT&CK" +[[rule.threat.technique]] +id = "T1210" +name = "Exploitation of Remote Services" +reference = "https://attack.mitre.org/techniques/T1210/" + + +[rule.threat.tactic] +id = "TA0008" +name = "Lateral Movement" +reference = "https://attack.mitre.org/tactics/TA0008/" + diff --git a/rules/integrations/lmd/lateral_movement_ml_spike_in_rdp_processes.toml b/rules/integrations/lmd/lateral_movement_ml_spike_in_rdp_processes.toml new file mode 100644 index 000000000..540ee068b --- /dev/null +++ b/rules/integrations/lmd/lateral_movement_ml_spike_in_rdp_processes.toml @@ -0,0 +1,44 @@ +[metadata] +creation_date = "2023/09/12" +integration = ["lmd"] +maturity = "production" +min_stack_comments = "New fields added: required_fields, related_integrations, setup" +min_stack_version = "8.3.0" +updated_date = "2023/09/21" + +[rule] +anomaly_threshold = 70 +author = ["Elastic"] +description = """ +A machine learning job has detected unusually high number of processes started in a single RDP session. Executing a +large number of processes remotely on other machines can be an indicator of lateral movement activity. +""" +from = "now-12h" +interval = "15m" +license = "Elastic License v2" +machine_learning_job_id = "lmd_high_sum_rdp_number_of_processes" +name = "Spike in Number of Processes in an RDP Session" +references = ["https://www.elastic.co/guide/en/security/current/prebuilt-ml-jobs.html"] +risk_score = 21 +rule_id = "19e9daf3-f5c5-4bc2-a9af-6b1e97098f03" +severity = "low" +tags = [ + "Use Case: Lateral Movement Detection", + "Rule Type: ML", + "Rule Type: Machine Learning", + "Tactic: Lateral Movement", +] +type = "machine_learning" +[[rule.threat]] +framework = "MITRE ATT&CK" +[[rule.threat.technique]] +id = "T1210" +name = "Exploitation of Remote Services" +reference = "https://attack.mitre.org/techniques/T1210/" + + +[rule.threat.tactic] +id = "TA0008" +name = "Lateral Movement" +reference = "https://attack.mitre.org/tactics/TA0008/" + diff --git a/rules/integrations/lmd/lateral_movement_ml_spike_in_remote_file_transfers.toml b/rules/integrations/lmd/lateral_movement_ml_spike_in_remote_file_transfers.toml new file mode 100644 index 000000000..28b32edd9 --- /dev/null +++ b/rules/integrations/lmd/lateral_movement_ml_spike_in_remote_file_transfers.toml @@ -0,0 +1,46 @@ +[metadata] +creation_date = "2023/09/13" +integration = ["lmd"] +maturity = "production" +min_stack_comments = "New fields added: required_fields, related_integrations, setup" +min_stack_version = "8.3.0" +updated_date = "2023/09/21" + +[rule] +anomaly_threshold = 70 +author = ["Elastic"] +description = """ +A machine learning job has detected an abnormal volume of remote files shared on the host indicating potential lateral +movement activity. One of the primary goals of attackers after gaining access to a network is to locate and exfiltrate +valuable information. Attackers might perform multiple small transfers to match normal egress activity in the network, +to evade detection. +""" +from = "now-90m" +interval = "15m" +license = "Elastic License v2" +machine_learning_job_id = "lmd_high_count_remote_file_transfer" +name = "Spike in Remote File Transfers" +references = ["https://www.elastic.co/guide/en/security/current/prebuilt-ml-jobs.html"] +risk_score = 21 +rule_id = "e9b0902b-c515-413b-b80b-a8dcebc81a66" +severity = "low" +tags = [ + "Use Case: Lateral Movement Detection", + "Rule Type: ML", + "Rule Type: Machine Learning", + "Tactic: Lateral Movement", +] +type = "machine_learning" +[[rule.threat]] +framework = "MITRE ATT&CK" +[[rule.threat.technique]] +id = "T1210" +name = "Exploitation of Remote Services" +reference = "https://attack.mitre.org/techniques/T1210/" + + +[rule.threat.tactic] +id = "TA0008" +name = "Lateral Movement" +reference = "https://attack.mitre.org/tactics/TA0008/" + diff --git a/rules/integrations/lmd/lateral_movement_ml_unusual_time_for_an_rdp_session.toml b/rules/integrations/lmd/lateral_movement_ml_unusual_time_for_an_rdp_session.toml new file mode 100644 index 000000000..db11bb949 --- /dev/null +++ b/rules/integrations/lmd/lateral_movement_ml_unusual_time_for_an_rdp_session.toml @@ -0,0 +1,45 @@ +[metadata] +creation_date = "2023/09/13" +integration = ["lmd"] +maturity = "production" +min_stack_comments = "New fields added: required_fields, related_integrations, setup" +min_stack_version = "8.3.0" +updated_date = "2023/09/21" + +[rule] +anomaly_threshold = 70 +author = ["Elastic"] +description = """ +A machine learning job has detected an RDP session started at an usual time or weekday. An RDP session at an unusual +time could be followed by other suspicious activities, so catching this is a good first step in detecting a larger +attack. +""" +from = "now-12h" +interval = "15m" +license = "Elastic License v2" +machine_learning_job_id = "lmd_unusual_time_weekday_rdp_session_start" +name = "Unusual Time or Day for an RDP Session" +references = ["https://www.elastic.co/guide/en/security/current/prebuilt-ml-jobs.html"] +risk_score = 21 +rule_id = "3f4e2dba-828a-452a-af35-fe29c5e78969" +severity = "low" +tags = [ + "Use Case: Lateral Movement Detection", + "Rule Type: ML", + "Rule Type: Machine Learning", + "Tactic: Lateral Movement", +] +type = "machine_learning" +[[rule.threat]] +framework = "MITRE ATT&CK" +[[rule.threat.technique]] +id = "T1210" +name = "Exploitation of Remote Services" +reference = "https://attack.mitre.org/techniques/T1210/" + + +[rule.threat.tactic] +id = "TA0008" +name = "Lateral Movement" +reference = "https://attack.mitre.org/tactics/TA0008/" + diff --git a/rules/integrations/lmd/lateral_movement_remote_file_creation_in_sensitive_directory.toml b/rules/integrations/lmd/lateral_movement_remote_file_creation_in_sensitive_directory.toml new file mode 100644 index 000000000..ae7845747 --- /dev/null +++ b/rules/integrations/lmd/lateral_movement_remote_file_creation_in_sensitive_directory.toml @@ -0,0 +1,57 @@ +[metadata] +creation_date = "2023/09/13" +integration = ["lmd","endpoint"] +maturity = "production" +min_stack_comments = "New fields added: required_fields, related_integrations, setup" +min_stack_version = "8.3.0" +updated_date = "2023/09/21" + +[rule] +author = ["Elastic"] +description = """ +Discovery of files created by a remote host on sensitive directories and folders. Remote file creation in these +directories could indicate a malicious binary or script trying to compromise the system. +""" +from = "now-10m" +index = ["logs-endpoint.events.*"] +interval = "5m" +language = "eql" +license = "Elastic License v2" +name = "Remote File Creation on a Sensitive Directory" +references = ["https://www.elastic.co/es/blog/remote-desktop-protocol-connections-elastic-security"] +risk_score = 47 +rule_id = "2377946d-0f01-4957-8812-6878985f515d" +severity = "medium" +tags = ["Domain: Endpoint", "Use Case: Lateral Movement Detection", "Tactic: Lateral Movement", "Data Source: Elastic Defend"] +timestamp_override = "event.ingested" +type = "eql" + +query = ''' +file where (event.action == "creation" or event.action == "modification") and +process.name:("System", "scp", "sshd", "smbd", "vsftpd", "sftp-server") and not +user.name:("SYSTEM", "root") and +(file.path : ("C*\\Users\\*\\AppData\\Roaming*", "C*\\Program*Files\\*", + "C*\\Windows\\*", "C*\\Windows\\System\\*", + "C*\\Windows\\System32\\*", "/etc/*", "/tmp*", + "/var/tmp*", "/home/*/.*", "/home/.*", "/usr/bin/*", + "/sbin/*", "/bin/*", "/usr/lib/*", "/usr/sbin/*", + "/usr/share/*", "/usr/local/*", "/var/lib/dpkg/*", + "/lib/systemd/*" + ) +) +''' + + +[[rule.threat]] +framework = "MITRE ATT&CK" +[[rule.threat.technique]] +id = "T1210" +name = "Exploitation of Remote Services" +reference = "https://attack.mitre.org/techniques/T1210/" + + +[rule.threat.tactic] +id = "TA0008" +name = "Lateral Movement" +reference = "https://attack.mitre.org/tactics/TA0008/" + diff --git a/tests/test_all_rules.py b/tests/test_all_rules.py index eaf7a86c7..458c0be1f 100644 --- a/tests/test_all_rules.py +++ b/tests/test_all_rules.py @@ -632,7 +632,8 @@ class TestRuleMetadata(BaseRuleTest): # checks if an index pattern exists if the package integration tag exists integration_string = "|".join(indices) if not re.search(rule_integration, integration_string): - if rule_integration == "windows" and re.search("winlog", integration_string): + if rule_integration == "windows" and re.search("winlog", integration_string) or \ + rule_integration in [*map(str.lower, definitions.MACHINE_LEARNING_PACKAGES)]: continue err_msg = f'{self.rule_str(rule)} {rule_integration} tag, index pattern missing.' failures.append(err_msg) @@ -658,7 +659,8 @@ class TestRuleMetadata(BaseRuleTest): ] if any([re.search("|".join(non_dataset_packages), i, re.IGNORECASE) for i in rule.contents.data.index]): - if not rule.contents.metadata.integration and rule.id not in ignore_ids: + if not rule.contents.metadata.integration and rule.id not in ignore_ids and \ + rule.contents.data.type not in definitions.MACHINE_LEARNING: err_msg = f'substrings {non_dataset_packages} found in '\ f'{self.rule_str(rule)} rule index patterns are {rule.contents.data.index},' \ f'but no integration tag found'