[New Rule] Adding Lateral Movement Rules from Advanced Analytic LMD Package (#3119)

* Adding Lateral Movement Detection rules

* added tags; adjusted tests; updated manifests and schemas

* added default value to build_integrations_schema

* combined analytic and non-dataset packages for related integrations

* adjusted machine learning definitions

* adjusted machine learning definitions

* removed splat for machine learning list due to 3.8 constraints

---------

Co-authored-by: terrancedejesus <terrance.dejesus@elastic.co>
Co-authored-by: Terrance DeJesus <99630311+terrancedejesus@users.noreply.github.com>

(cherry picked from commit 747ee7d593)

detection_rules/devtools.py

@@ -1236,14 +1236,19 @@ def build_integration_manifests(overwrite: bool, integration: str):
 
 @integrations_group.command('build-schemas')
 @click.option('--overwrite', '-o', is_flag=True, help="Overwrite the entire integrations-schema.json.gz file")
-def build_integration_schemas(overwrite: bool):
+@click.option('--integration', '-i', type=str,
+              help="Adds a single integration schema to the integrations-schema.json.gz file")
+def build_integration_schemas(overwrite: bool, integration: str):
     """Builds consolidated integrations schemas file."""
     click.echo("Building integration schemas...")
 
     start_time = time.perf_counter()
-    build_integrations_schemas(overwrite)
-    end_time = time.perf_counter()
-    click.echo(f"Time taken to generate schemas: {(end_time - start_time)/60:.2f} minutes")
+    if integration:
+        build_integrations_schemas(overwrite=False, integration=integration)
+    else:
+        build_integrations_schemas(overwrite=overwrite)
+        end_time = time.perf_counter()
+        click.echo(f"Time taken to generate schemas: {(end_time - start_time)/60:.2f} minutes")
 
 
 @integrations_group.command('show-latest-compatible')

detection_rules/integrations.py

@@ -47,12 +47,13 @@ class IntegrationManifestSchema(Schema):
     description = fields.Str(required=True)
     download = fields.Str(required=True)
     conditions = fields.Dict(required=True)
-    policy_templates = fields.List(fields.Dict, required=True)
+    policy_templates = fields.List(fields.Dict)
     owner = fields.Dict(required=False)
 
     @post_load
     def transform_policy_template(self, data, **kwargs):
-        data["policy_templates"] = [policy["name"] for policy in data["policy_templates"]]
+        if "policy_templates" in data:
+            data["policy_templates"] = [policy["name"] for policy in data["policy_templates"]]
         return data
 
 
@@ -93,21 +94,30 @@ def build_integrations_manifest(overwrite: bool, rule_integrations: list = [], i
     print(f"final integrations manifests dumped: {MANIFEST_FILE_PATH}")
 
 
-def build_integrations_schemas(overwrite: bool) -> None:
+def build_integrations_schemas(overwrite: bool, integration: str = None) -> None:
     """Builds a new local copy of integration-schemas.json.gz from EPR integrations."""
 
-    final_integration_schemas = {}
     saved_integration_schemas = {}
 
     # Check if the file already exists and handle accordingly
     if overwrite and SCHEMA_FILE_PATH.exists():
         SCHEMA_FILE_PATH.unlink()
+        final_integration_schemas = {}
     elif SCHEMA_FILE_PATH.exists():
-        saved_integration_schemas = load_integrations_schemas()
+        final_integration_schemas = load_integrations_schemas()
+    else:
+        final_integration_schemas = {}
 
     # Load the integration manifests
     integration_manifests = load_integrations_manifests()
 
+    # if a single integration is specified, only process that integration
+    if integration:
+        if integration in integration_manifests:
+            integration_manifests = {integration: integration_manifests[integration]}
+        else:
+            raise ValueError(f"Integration {integration} not found in manifest.")
+
     # Loop through the packages and versions
     for package, versions in integration_manifests.items():
         print(f"processing {package}")

detection_rules/rule.py

@@ -1024,8 +1024,10 @@ class TOMLRuleContents(BaseRuleContents, MarshmallowDataclassMixin):
 
                             # if integration is not a policy template remove
                             if package["version"]:
-                                policy_templates = packages_manifest[
-                                    package["package"]][package["version"].strip("^")]["policy_templates"]
+                                version_data = packages_manifest.get(package["package"],
+                                                                     {}).get(package["version"].strip("^"), {})
+                                policy_templates = version_data.get("policy_templates", [])
+
                                 if package["integration"] not in policy_templates:
                                     del package["integration"]
 
@@ -1131,7 +1133,9 @@ class TOMLRuleContents(BaseRuleContents, MarshmallowDataclassMixin):
             rule_integrations = meta.get("integration", [])
             if rule_integrations:
                 for integration in rule_integrations:
-                    if integration in definitions.NON_DATASET_PACKAGES or isinstance(data, MachineLearningRuleData):
+                    ineligible_integrations = definitions.NON_DATASET_PACKAGES + \
+                        [*map(str.lower, definitions.MACHINE_LEARNING_PACKAGES)]
+                    if integration in ineligible_integrations or isinstance(data, MachineLearningRuleData):
                         packaged_integrations.append({"package": integration, "integration": None})
 
         for value in sorted(datasets):

detection_rules/schemas/definitions.py

@@ -125,6 +125,7 @@ EXPECTED_RULE_TAGS = [
     'Use Case: Vulnerability'
 ]
 
+MACHINE_LEARNING_PACKAGES = ['LMD', 'DGA', 'DED', 'ProblemChild', 'Beaconing']
 
 NonEmptyStr = NewType('NonEmptyStr', str, validate=validate.Length(min=1))
 TimeUnits = Literal['s', 'm', 'h']
@@ -159,5 +160,6 @@ UUIDString = NewType('UUIDString', str, validate=validate.Regexp(UUID_PATTERN))
 BuildingBlockType = Literal['default']
 
 # experimental machine learning features and releases
-MachineLearningType = Literal['DGA', 'ProblemChild']
-MachineLearningTypeLower = Literal['dga', 'problemchild']
+MachineLearningType = getattr(Literal, '__getitem__')(tuple(MACHINE_LEARNING_PACKAGES))  # noqa: E999
+MachineLearningTypeLower = getattr(Literal, '__getitem__')(
+    tuple(map(str.lower, MACHINE_LEARNING_PACKAGES)))  # noqa: E999

rules/integrations/lmd/lateral_movement_malicious_remote_file_creation.toml

@@ -0,0 +1,45 @@
+[metadata]
+creation_date = "2023/09/13"
+integration = ["lmd","endpoint"]
+maturity = "production"
+min_stack_comments = "New fields added: required_fields, related_integrations, setup"
+min_stack_version = "8.3.0"
+updated_date = "2023/09/21"
+
+[rule]
+author = ["Elastic"]
+description = "Malicious remote file creation, which can be an indicator of lateral movement activity."
+from = "now-10m"
+index = ["logs-endpoint.events.*"]
+interval = "5m"
+language = "eql"
+license = "Elastic License v2"
+name = "Malicious Remote File Creation"
+references = ["https://www.elastic.co/es/blog/remote-desktop-protocol-connections-elastic-security"]
+risk_score = 99
+rule_id = "301571f3-b316-4969-8dd0-7917410030d3"
+severity = "critical"
+tags = ["Domain: Endpoint", "Use Case: Lateral Movement Detection", "Tactic: Lateral Movement", "Data Source: Elastic Defend"]
+type = "eql"
+
+query = '''
+sequence by host.name
+[file where event.action == "creation" and process.name : ("System", "scp", "sshd", "smbd", "vsftpd", "sftp-server")]
+[file where event.category == "malware" or event.category == "intrusion_detection"
+and process.name:("System", "scp", "sshd", "smbd", "vsftpd", "sftp-server")]
+'''
+
+
+[[rule.threat]]
+framework = "MITRE ATT&CK"
+[[rule.threat.technique]]
+id = "T1210"
+name = "Exploitation of Remote Services"
+reference = "https://attack.mitre.org/techniques/T1210/"
+
+
+[rule.threat.tactic]
+id = "TA0008"
+name = "Lateral Movement"
+reference = "https://attack.mitre.org/tactics/TA0008/"
+

rules/integrations/lmd/lateral_movement_ml_high_mean_rdp_process_args.toml

@@ -0,0 +1,45 @@
+[metadata]
+creation_date = "2023/09/13"
+integration = ["lmd"]
+maturity = "production"
+min_stack_comments = "New fields added: required_fields, related_integrations, setup"
+min_stack_version = "8.3.0"
+updated_date = "2023/09/21"
+
+[rule]
+anomaly_threshold = 70
+author = ["Elastic"]
+description = """
+A machine learning job has detected unusually high number of process arguments in an RDP session. Executing
+sophisticated attacks such as lateral movement can involve the use of complex commands, obfuscation mechanisms,
+redirection and piping, which in turn increases the number of arguments in a command.
+"""
+from = "now-12h"
+interval = "15m"
+license = "Elastic License v2"
+machine_learning_job_id = "lmd_high_mean_rdp_process_args"
+name = "High Mean of Process Arguments in an RDP Session"
+references = ["https://www.elastic.co/guide/en/security/current/prebuilt-ml-jobs.html"]
+risk_score = 21
+rule_id = "36c48a0c-c63a-4cbc-aee1-8cac87db31a9"
+severity = "low"
+tags = [
+    "Use Case: Lateral Movement Detection",
+    "Rule Type: ML",
+    "Rule Type: Machine Learning",
+    "Tactic: Lateral Movement",
+]
+type = "machine_learning"
+[[rule.threat]]
+framework = "MITRE ATT&CK"
+[[rule.threat.technique]]
+id = "T1210"
+name = "Exploitation of Remote Services"
+reference = "https://attack.mitre.org/techniques/T1210/"
+
+
+[rule.threat.tactic]
+id = "TA0008"
+name = "Lateral Movement"
+reference = "https://attack.mitre.org/tactics/TA0008/"
+

rules/integrations/lmd/lateral_movement_ml_high_mean_rdp_session_duration.toml

@@ -0,0 +1,45 @@
+[metadata]
+creation_date = "2023/09/12"
+integration = ["lmd"]
+maturity = "production"
+min_stack_comments = "New fields added: required_fields, related_integrations, setup"
+min_stack_version = "8.3.0"
+updated_date = "2023/09/21"
+
+[rule]
+anomaly_threshold = 70
+author = ["Elastic"]
+description = """
+A machine learning job has detected unusually high mean of RDP session duration. Long RDP sessions can be used to evade
+detection mechanisms via session persistence, and might be used to perform tasks such as lateral movement, that might
+require uninterrupted access to a compromised machine.
+"""
+from = "now-12h"
+interval = "15m"
+license = "Elastic License v2"
+machine_learning_job_id = "lmd_high_mean_rdp_session_duration"
+name = "High Mean of RDP Session Duration"
+references = ["https://www.elastic.co/guide/en/security/current/prebuilt-ml-jobs.html"]
+risk_score = 21
+rule_id = "a74c60cb-70ee-4629-a127-608ead14ebf1"
+severity = "low"
+tags = [
+    "Use Case: Lateral Movement Detection",
+    "Rule Type: ML",
+    "Rule Type: Machine Learning",
+    "Tactic: Lateral Movement",
+]
+type = "machine_learning"
+[[rule.threat]]
+framework = "MITRE ATT&CK"
+[[rule.threat.technique]]
+id = "T1210"
+name = "Exploitation of Remote Services"
+reference = "https://attack.mitre.org/techniques/T1210/"
+
+
+[rule.threat.tactic]
+id = "TA0008"
+name = "Lateral Movement"
+reference = "https://attack.mitre.org/tactics/TA0008/"
+

rules/integrations/lmd/lateral_movement_ml_high_remote_file_size.toml

@@ -0,0 +1,46 @@
+[metadata]
+creation_date = "2023/09/13"
+integration = ["lmd"]
+maturity = "production"
+min_stack_comments = "New fields added: required_fields, related_integrations, setup"
+min_stack_version = "8.3.0"
+updated_date = "2023/09/21"
+
+[rule]
+anomaly_threshold = 70
+author = ["Elastic"]
+description = """
+A machine learning job has detected an unusually high file size shared by a remote host indicating potential lateral
+movement activity. One of the primary goals of attackers after gaining access to a network is to locate and exfiltrate
+valuable information. Instead of multiple small transfers that can raise alarms, attackers might choose to bundle data
+into a single large file transfer.
+"""
+from = "now-90m"
+interval = "15m"
+license = "Elastic License v2"
+machine_learning_job_id = "lmd_high_file_size_remote_file_transfer"
+name = "Unusual Remote File Size"
+references = ["https://www.elastic.co/guide/en/security/current/prebuilt-ml-jobs.html"]
+risk_score = 21
+rule_id = "0678bc9c-b71a-433b-87e6-2f664b6b3131"
+severity = "low"
+tags = [
+    "Use Case: Lateral Movement Detection",
+    "Rule Type: ML",
+    "Rule Type: Machine Learning",
+    "Tactic: Lateral Movement",
+]
+type = "machine_learning"
+[[rule.threat]]
+framework = "MITRE ATT&CK"
+[[rule.threat.technique]]
+id = "T1210"
+name = "Exploitation of Remote Services"
+reference = "https://attack.mitre.org/techniques/T1210/"
+
+
+[rule.threat.tactic]
+id = "TA0008"
+name = "Lateral Movement"
+reference = "https://attack.mitre.org/tactics/TA0008/"
+

rules/integrations/lmd/lateral_movement_ml_high_variance_rdp_session_duration.toml

@@ -0,0 +1,45 @@
+[metadata]
+creation_date = "2023/09/13"
+integration = ["lmd"]
+maturity = "production"
+min_stack_comments = "New fields added: required_fields, related_integrations, setup"
+min_stack_version = "8.3.0"
+updated_date = "2023/09/21"
+
+[rule]
+anomaly_threshold = 70
+author = ["Elastic"]
+description = """
+A machine learning job has detected unusually high variance of RDP session duration. Long RDP sessions can be used to
+evade detection mechanisms via session persistence, and might be used to perform tasks such as lateral movement, that
+might require uninterrupted access to a compromised machine.
+"""
+from = "now-12h"
+interval = "15m"
+license = "Elastic License v2"
+machine_learning_job_id = "lmd_high_var_rdp_session_duration"
+name = "High Variance in RDP Session Duration"
+references = ["https://www.elastic.co/guide/en/security/current/prebuilt-ml-jobs.html"]
+risk_score = 21
+rule_id = "a8d35ca0-ad8d-48a9-9f6c-553622dca61a"
+severity = "low"
+tags = [
+    "Use Case: Lateral Movement Detection",
+    "Rule Type: ML",
+    "Rule Type: Machine Learning",
+    "Tactic: Lateral Movement",
+]
+type = "machine_learning"
+[[rule.threat]]
+framework = "MITRE ATT&CK"
+[[rule.threat.technique]]
+id = "T1210"
+name = "Exploitation of Remote Services"
+reference = "https://attack.mitre.org/techniques/T1210/"
+
+
+[rule.threat.tactic]
+id = "TA0008"
+name = "Lateral Movement"
+reference = "https://attack.mitre.org/tactics/TA0008/"
+

rules/integrations/lmd/lateral_movement_ml_rare_remote_file_directory.toml

@@ -0,0 +1,45 @@
+[metadata]
+creation_date = "2023/09/12"
+integration = ["lmd"]
+maturity = "production"
+min_stack_comments = "New fields added: required_fields, related_integrations, setup"
+min_stack_version = "8.3.0"
+updated_date = "2023/09/21"
+
+[rule]
+anomaly_threshold = 70
+author = ["Elastic"]
+description = """
+An anomaly detection job has detected a remote file transfer on an unusual directory indicating a potential lateral
+movement activity on the host. Many Security solutions monitor well-known directories for suspicious activities, so
+attackers might use less common directories to bypass monitoring.
+"""
+from = "now-90m"
+interval = "15m"
+license = "Elastic License v2"
+machine_learning_job_id = "lmd_rare_file_path_remote_transfer"
+name = "Unusual Remote File Directory"
+references = ["https://www.elastic.co/guide/en/security/current/prebuilt-ml-jobs.html"]
+risk_score = 21
+rule_id = "be4c5aed-90f5-4221-8bd5-7ab3a4334751"
+severity = "low"
+tags = [
+    "Use Case: Lateral Movement Detection",
+    "Rule Type: ML",
+    "Rule Type: Machine Learning",
+    "Tactic: Lateral Movement",
+]
+type = "machine_learning"
+[[rule.threat]]
+framework = "MITRE ATT&CK"
+[[rule.threat.technique]]
+id = "T1210"
+name = "Exploitation of Remote Services"
+reference = "https://attack.mitre.org/techniques/T1210/"
+
+
+[rule.threat.tactic]
+id = "TA0008"
+name = "Lateral Movement"
+reference = "https://attack.mitre.org/tactics/TA0008/"
+

rules/integrations/lmd/lateral_movement_ml_rare_remote_file_extension.toml

@@ -0,0 +1,44 @@
+[metadata]
+creation_date = "2023/09/13"
+integration = ["lmd"]
+maturity = "production"
+min_stack_comments = "New fields added: required_fields, related_integrations, setup"
+min_stack_version = "8.3.0"
+updated_date = "2023/09/21"
+
+[rule]
+anomaly_threshold = 70
+author = ["Elastic"]
+description = """
+An anomaly detection job has detected a remote file transfer with a rare extension, which could indicate potential
+lateral movement activity on the host.
+"""
+from = "now-90m"
+interval = "15m"
+license = "Elastic License v2"
+machine_learning_job_id = "lmd_rare_file_extension_remote_transfer"
+name = "Unusual Remote File Extension"
+references = ["https://www.elastic.co/guide/en/security/current/prebuilt-ml-jobs.html"]
+risk_score = 21
+rule_id = "814d96c7-2068-42aa-ba8e-fe0ddd565e2e"
+severity = "low"
+tags = [
+    "Use Case: Lateral Movement Detection",
+    "Rule Type: ML",
+    "Rule Type: Machine Learning",
+    "Tactic: Lateral Movement",
+]
+type = "machine_learning"
+[[rule.threat]]
+framework = "MITRE ATT&CK"
+[[rule.threat.technique]]
+id = "T1210"
+name = "Exploitation of Remote Services"
+reference = "https://attack.mitre.org/techniques/T1210/"
+
+
+[rule.threat.tactic]
+id = "TA0008"
+name = "Lateral Movement"
+reference = "https://attack.mitre.org/tactics/TA0008/"
+

rules/integrations/lmd/lateral_movement_ml_spike_in_connections_from_a_source_ip.toml

@@ -0,0 +1,45 @@
+[metadata]
+creation_date = "2023/09/13"
+integration = ["lmd"]
+maturity = "production"
+min_stack_comments = "New fields added: required_fields, related_integrations, setup"
+min_stack_version = "8.3.0"
+updated_date = "2023/09/21"
+
+[rule]
+anomaly_threshold = 70
+author = ["Elastic"]
+description = """
+A machine learning job has detected a high count of destination IPs establishing an RDP connection with a single source
+IP. Once an attacker has gained access to one system, they might attempt to access more in the network in search of
+valuable assets, data, or further access points.
+"""
+from = "now-12h"
+interval = "15m"
+license = "Elastic License v2"
+machine_learning_job_id = "lmd_high_rdp_distinct_count_destination_ip_for_source"
+name = "Spike in Number of Connections Made from a Source IP"
+references = ["https://www.elastic.co/guide/en/security/current/prebuilt-ml-jobs.html"]
+risk_score = 21
+rule_id = "3e0561b5-3fac-4461-84cc-19163b9aaa61"
+severity = "low"
+tags = [
+    "Use Case: Lateral Movement Detection",
+    "Rule Type: ML",
+    "Rule Type: Machine Learning",
+    "Tactic: Lateral Movement",
+]
+type = "machine_learning"
+[[rule.threat]]
+framework = "MITRE ATT&CK"
+[[rule.threat.technique]]
+id = "T1210"
+name = "Exploitation of Remote Services"
+reference = "https://attack.mitre.org/techniques/T1210/"
+
+
+[rule.threat.tactic]
+id = "TA0008"
+name = "Lateral Movement"
+reference = "https://attack.mitre.org/tactics/TA0008/"
+

rules/integrations/lmd/lateral_movement_ml_spike_in_connections_to_a_destination_ip.toml

@@ -0,0 +1,45 @@
+[metadata]
+creation_date = "2023/09/13"
+integration = ["lmd"]
+maturity = "production"
+min_stack_comments = "New fields added: required_fields, related_integrations, setup"
+min_stack_version = "8.3.0"
+updated_date = "2023/09/21"
+
+[rule]
+anomaly_threshold = 70
+author = ["Elastic"]
+description = """
+A machine learning job has detected a high count of source IPs establishing an RDP connection with a single destination
+IP. Attackers might use multiple compromised systems to attack a target to ensure redundancy in case a source IP gets
+detected and blocked.
+"""
+from = "now-12h"
+interval = "15m"
+license = "Elastic License v2"
+machine_learning_job_id = "lmd_high_rdp_distinct_count_source_ip_for_destination"
+name = "Spike in Number of Connections Made to a Destination IP"
+references = ["https://www.elastic.co/guide/en/security/current/prebuilt-ml-jobs.html"]
+risk_score = 21
+rule_id = "18a5dd9a-e3fa-4996-99b1-ae533b8f27fc"
+severity = "low"
+tags = [
+    "Use Case: Lateral Movement Detection",
+    "Rule Type: ML",
+    "Rule Type: Machine Learning",
+    "Tactic: Lateral Movement",
+]
+type = "machine_learning"
+[[rule.threat]]
+framework = "MITRE ATT&CK"
+[[rule.threat.technique]]
+id = "T1210"
+name = "Exploitation of Remote Services"
+reference = "https://attack.mitre.org/techniques/T1210/"
+
+
+[rule.threat.tactic]
+id = "TA0008"
+name = "Lateral Movement"
+reference = "https://attack.mitre.org/tactics/TA0008/"
+

rules/integrations/lmd/lateral_movement_ml_spike_in_rdp_processes.toml

@@ -0,0 +1,44 @@
+[metadata]
+creation_date = "2023/09/12"
+integration = ["lmd"]
+maturity = "production"
+min_stack_comments = "New fields added: required_fields, related_integrations, setup"
+min_stack_version = "8.3.0"
+updated_date = "2023/09/21"
+
+[rule]
+anomaly_threshold = 70
+author = ["Elastic"]
+description = """
+A machine learning job has detected unusually high number of processes started in a single RDP session. Executing a
+large number of processes remotely on other machines can be an indicator of lateral movement activity.
+"""
+from = "now-12h"
+interval = "15m"
+license = "Elastic License v2"
+machine_learning_job_id = "lmd_high_sum_rdp_number_of_processes"
+name = "Spike in Number of Processes in an RDP Session"
+references = ["https://www.elastic.co/guide/en/security/current/prebuilt-ml-jobs.html"]
+risk_score = 21
+rule_id = "19e9daf3-f5c5-4bc2-a9af-6b1e97098f03"
+severity = "low"
+tags = [
+    "Use Case: Lateral Movement Detection",
+    "Rule Type: ML",
+    "Rule Type: Machine Learning",
+    "Tactic: Lateral Movement",
+]
+type = "machine_learning"
+[[rule.threat]]
+framework = "MITRE ATT&CK"
+[[rule.threat.technique]]
+id = "T1210"
+name = "Exploitation of Remote Services"
+reference = "https://attack.mitre.org/techniques/T1210/"
+
+
+[rule.threat.tactic]
+id = "TA0008"
+name = "Lateral Movement"
+reference = "https://attack.mitre.org/tactics/TA0008/"
+

rules/integrations/lmd/lateral_movement_ml_spike_in_remote_file_transfers.toml

@@ -0,0 +1,46 @@
+[metadata]
+creation_date = "2023/09/13"
+integration = ["lmd"]
+maturity = "production"
+min_stack_comments = "New fields added: required_fields, related_integrations, setup"
+min_stack_version = "8.3.0"
+updated_date = "2023/09/21"
+
+[rule]
+anomaly_threshold = 70
+author = ["Elastic"]
+description = """
+A machine learning job has detected an abnormal volume of remote files shared on the host indicating potential lateral
+movement activity. One of the primary goals of attackers after gaining access to a network is to locate and exfiltrate
+valuable information. Attackers might perform multiple small transfers to match normal egress activity in the network,
+to evade detection.
+"""
+from = "now-90m"
+interval = "15m"
+license = "Elastic License v2"
+machine_learning_job_id = "lmd_high_count_remote_file_transfer"
+name = "Spike in Remote File Transfers"
+references = ["https://www.elastic.co/guide/en/security/current/prebuilt-ml-jobs.html"]
+risk_score = 21
+rule_id = "e9b0902b-c515-413b-b80b-a8dcebc81a66"
+severity = "low"
+tags = [
+    "Use Case: Lateral Movement Detection",
+    "Rule Type: ML",
+    "Rule Type: Machine Learning",
+    "Tactic: Lateral Movement",
+]
+type = "machine_learning"
+[[rule.threat]]
+framework = "MITRE ATT&CK"
+[[rule.threat.technique]]
+id = "T1210"
+name = "Exploitation of Remote Services"
+reference = "https://attack.mitre.org/techniques/T1210/"
+
+
+[rule.threat.tactic]
+id = "TA0008"
+name = "Lateral Movement"
+reference = "https://attack.mitre.org/tactics/TA0008/"
+

rules/integrations/lmd/lateral_movement_ml_unusual_time_for_an_rdp_session.toml

@@ -0,0 +1,45 @@
+[metadata]
+creation_date = "2023/09/13"
+integration = ["lmd"]
+maturity = "production"
+min_stack_comments = "New fields added: required_fields, related_integrations, setup"
+min_stack_version = "8.3.0"
+updated_date = "2023/09/21"
+
+[rule]
+anomaly_threshold = 70
+author = ["Elastic"]
+description = """
+A machine learning job has detected an RDP session started at an usual time or weekday. An RDP session at an unusual
+time could be followed by other suspicious activities, so catching this is a good first step in detecting a larger
+attack.
+"""
+from = "now-12h"
+interval = "15m"
+license = "Elastic License v2"
+machine_learning_job_id = "lmd_unusual_time_weekday_rdp_session_start"
+name = "Unusual Time or Day for an RDP Session"
+references = ["https://www.elastic.co/guide/en/security/current/prebuilt-ml-jobs.html"]
+risk_score = 21
+rule_id = "3f4e2dba-828a-452a-af35-fe29c5e78969"
+severity = "low"
+tags = [
+    "Use Case: Lateral Movement Detection",
+    "Rule Type: ML",
+    "Rule Type: Machine Learning",
+    "Tactic: Lateral Movement",
+]
+type = "machine_learning"
+[[rule.threat]]
+framework = "MITRE ATT&CK"
+[[rule.threat.technique]]
+id = "T1210"
+name = "Exploitation of Remote Services"
+reference = "https://attack.mitre.org/techniques/T1210/"
+
+
+[rule.threat.tactic]
+id = "TA0008"
+name = "Lateral Movement"
+reference = "https://attack.mitre.org/tactics/TA0008/"
+

rules/integrations/lmd/lateral_movement_remote_file_creation_in_sensitive_directory.toml

@@ -0,0 +1,57 @@
+[metadata]
+creation_date = "2023/09/13"
+integration = ["lmd","endpoint"]
+maturity = "production"
+min_stack_comments = "New fields added: required_fields, related_integrations, setup"
+min_stack_version = "8.3.0"
+updated_date = "2023/09/21"
+
+[rule]
+author = ["Elastic"]
+description = """
+Discovery of files created by a remote host on sensitive directories and folders. Remote file creation in these
+directories could indicate a malicious binary or script trying to compromise the system.
+"""
+from = "now-10m"
+index = ["logs-endpoint.events.*"]
+interval = "5m"
+language = "eql"
+license = "Elastic License v2"
+name = "Remote File Creation on a Sensitive Directory"
+references = ["https://www.elastic.co/es/blog/remote-desktop-protocol-connections-elastic-security"]
+risk_score = 47
+rule_id = "2377946d-0f01-4957-8812-6878985f515d"
+severity = "medium"
+tags = ["Domain: Endpoint", "Use Case: Lateral Movement Detection", "Tactic: Lateral Movement", "Data Source: Elastic Defend"]
+timestamp_override = "event.ingested"
+type = "eql"
+
+query = '''
+file where (event.action == "creation" or event.action == "modification") and
+process.name:("System", "scp", "sshd", "smbd", "vsftpd", "sftp-server") and not
+user.name:("SYSTEM", "root") and
+(file.path : ("C*\\Users\\*\\AppData\\Roaming*", "C*\\Program*Files\\*",
+    "C*\\Windows\\*", "C*\\Windows\\System\\*",
+    "C*\\Windows\\System32\\*", "/etc/*", "/tmp*",
+    "/var/tmp*", "/home/*/.*", "/home/.*", "/usr/bin/*",
+    "/sbin/*", "/bin/*", "/usr/lib/*", "/usr/sbin/*",
+    "/usr/share/*", "/usr/local/*", "/var/lib/dpkg/*",
+    "/lib/systemd/*"
+    )
+)
+'''
+
+
+[[rule.threat]]
+framework = "MITRE ATT&CK"
+[[rule.threat.technique]]
+id = "T1210"
+name = "Exploitation of Remote Services"
+reference = "https://attack.mitre.org/techniques/T1210/"
+
+
+[rule.threat.tactic]
+id = "TA0008"
+name = "Lateral Movement"
+reference = "https://attack.mitre.org/tactics/TA0008/"
+

tests/test_all_rules.py

@@ -632,7 +632,8 @@ class TestRuleMetadata(BaseRuleTest):
                     # checks if an index pattern exists if the package integration tag exists
                     integration_string = "|".join(indices)
                     if not re.search(rule_integration, integration_string):
-                        if rule_integration == "windows" and re.search("winlog", integration_string):
+                        if rule_integration == "windows" and re.search("winlog", integration_string) or \
+                                rule_integration in [*map(str.lower, definitions.MACHINE_LEARNING_PACKAGES)]:
                             continue
                         err_msg = f'{self.rule_str(rule)} {rule_integration} tag, index pattern missing.'
                         failures.append(err_msg)
@@ -658,7 +659,8 @@ class TestRuleMetadata(BaseRuleTest):
                     ]
                     if any([re.search("|".join(non_dataset_packages), i, re.IGNORECASE)
                             for i in rule.contents.data.index]):
-                        if not rule.contents.metadata.integration and rule.id not in ignore_ids:
+                        if not rule.contents.metadata.integration and rule.id not in ignore_ids and \
+                                rule.contents.data.type not in definitions.MACHINE_LEARNING:
                             err_msg = f'substrings {non_dataset_packages} found in '\
                                       f'{self.rule_str(rule)} rule index patterns are {rule.contents.data.index},' \
                                       f'but no integration tag found'