[New Rule] Adding Lateral Movement Rules from Advanced Analytic LMD Package (#3119)

* Adding Lateral Movement Detection rules

* added tags; adjusted tests; updated manifests and schemas

* added default value to build_integrations_schema

* combined analytic and non-dataset packages for related integrations

* adjusted machine learning definitions

* adjusted machine learning definitions

* removed splat for machine learning list due to 3.8 constraints

---------

Co-authored-by: terrancedejesus <terrance.dejesus@elastic.co>
Co-authored-by: Terrance DeJesus <99630311+terrancedejesus@users.noreply.github.com>

(cherry picked from commit 747ee7d593)

detection_rules/devtools.py

@@ -1236,14 +1236,19 @@ def build_integration_manifests(overwrite: bool, integration: str):
 
 @integrations_group.command('build-schemas')
 @click.option('--overwrite', '-o', is_flag=True, help="Overwrite the entire integrations-schema.json.gz file")
-def build_integration_schemas(overwrite: bool):
+@click.option('--integration', '-i', type=str,
+              help="Adds a single integration schema to the integrations-schema.json.gz file")
+def build_integration_schemas(overwrite: bool, integration: str):
     """Builds consolidated integrations schemas file."""
     click.echo("Building integration schemas...")
 
     start_time = time.perf_counter()
-    build_integrations_schemas(overwrite)
-    end_time = time.perf_counter()
-    click.echo(f"Time taken to generate schemas: {(end_time - start_time)/60:.2f} minutes")
+    if integration:
+        build_integrations_schemas(overwrite=False, integration=integration)
+    else:
+        build_integrations_schemas(overwrite=overwrite)
+        end_time = time.perf_counter()
+        click.echo(f"Time taken to generate schemas: {(end_time - start_time)/60:.2f} minutes")
 
 
 @integrations_group.command('show-latest-compatible')

detection_rules/integrations.py

@@ -47,12 +47,13 @@ class IntegrationManifestSchema(Schema):
     description = fields.Str(required=True)
     download = fields.Str(required=True)
     conditions = fields.Dict(required=True)
-    policy_templates = fields.List(fields.Dict, required=True)
+    policy_templates = fields.List(fields.Dict)
     owner = fields.Dict(required=False)
 
     @post_load
     def transform_policy_template(self, data, **kwargs):
-        data["policy_templates"] = [policy["name"] for policy in data["policy_templates"]]
+        if "policy_templates" in data:
+            data["policy_templates"] = [policy["name"] for policy in data["policy_templates"]]
         return data
 
 
@@ -93,21 +94,30 @@ def build_integrations_manifest(overwrite: bool, rule_integrations: list = [], i
     print(f"final integrations manifests dumped: {MANIFEST_FILE_PATH}")
 
 
-def build_integrations_schemas(overwrite: bool) -> None:
+def build_integrations_schemas(overwrite: bool, integration: str = None) -> None:
     """Builds a new local copy of integration-schemas.json.gz from EPR integrations."""
 
-    final_integration_schemas = {}
     saved_integration_schemas = {}
 
     # Check if the file already exists and handle accordingly
     if overwrite and SCHEMA_FILE_PATH.exists():
         SCHEMA_FILE_PATH.unlink()
+        final_integration_schemas = {}
     elif SCHEMA_FILE_PATH.exists():
-        saved_integration_schemas = load_integrations_schemas()
+        final_integration_schemas = load_integrations_schemas()
+    else:
+        final_integration_schemas = {}
 
     # Load the integration manifests
     integration_manifests = load_integrations_manifests()
 
+    # if a single integration is specified, only process that integration
+    if integration:
+        if integration in integration_manifests:
+            integration_manifests = {integration: integration_manifests[integration]}
+        else:
+            raise ValueError(f"Integration {integration} not found in manifest.")
+
     # Loop through the packages and versions
     for package, versions in integration_manifests.items():
         print(f"processing {package}")

detection_rules/rule.py

@@ -1024,8 +1024,10 @@ class TOMLRuleContents(BaseRuleContents, MarshmallowDataclassMixin):
 
                             # if integration is not a policy template remove
                             if package["version"]:
-                                policy_templates = packages_manifest[
-                                    package["package"]][package["version"].strip("^")]["policy_templates"]
+                                version_data = packages_manifest.get(package["package"],
+                                                                     {}).get(package["version"].strip("^"), {})
+                                policy_templates = version_data.get("policy_templates", [])
+
                                 if package["integration"] not in policy_templates:
                                     del package["integration"]
 
@@ -1131,7 +1133,9 @@ class TOMLRuleContents(BaseRuleContents, MarshmallowDataclassMixin):
             rule_integrations = meta.get("integration", [])
             if rule_integrations:
                 for integration in rule_integrations:
-                    if integration in definitions.NON_DATASET_PACKAGES or isinstance(data, MachineLearningRuleData):
+                    ineligible_integrations = definitions.NON_DATASET_PACKAGES + \
+                        [*map(str.lower, definitions.MACHINE_LEARNING_PACKAGES)]
+                    if integration in ineligible_integrations or isinstance(data, MachineLearningRuleData):
                         packaged_integrations.append({"package": integration, "integration": None})
 
         for value in sorted(datasets):

detection_rules/schemas/definitions.py

@@ -125,6 +125,7 @@ EXPECTED_RULE_TAGS = [
     'Use Case: Vulnerability'
 ]
 
+MACHINE_LEARNING_PACKAGES = ['LMD', 'DGA', 'DED', 'ProblemChild', 'Beaconing']
 
 NonEmptyStr = NewType('NonEmptyStr', str, validate=validate.Length(min=1))
 TimeUnits = Literal['s', 'm', 'h']
@@ -159,5 +160,6 @@ UUIDString = NewType('UUIDString', str, validate=validate.Regexp(UUID_PATTERN))
 BuildingBlockType = Literal['default']
 
 # experimental machine learning features and releases
-MachineLearningType = Literal['DGA', 'ProblemChild']
-MachineLearningTypeLower = Literal['dga', 'problemchild']
+MachineLearningType = getattr(Literal, '__getitem__')(tuple(MACHINE_LEARNING_PACKAGES))  # noqa: E999
+MachineLearningTypeLower = getattr(Literal, '__getitem__')(
+    tuple(map(str.lower, MACHINE_LEARNING_PACKAGES)))  # noqa: E999