From 9bddabf8e951abd8ca1beed2d80c440c9f3838bb Mon Sep 17 00:00:00 2001 From: Ross Wolf <31489089+rw-access@users.noreply.github.com> Date: Mon, 19 Jul 2021 12:52:44 -0600 Subject: [PATCH] Add optional integration field to the schema (#1359) (cherry picked from commit 816e31cd3824427407a90ca625886535ce9be1c9) --- detection_rules/mixins.py | 26 +- detection_rules/rule.py | 1 + etc/api_schemas/7.14/7.14.base.json | 310 +++++++++++++ etc/api_schemas/7.14/7.14.eql.json | 321 ++++++++++++++ .../7.14/7.14.machine_learning.json | 323 ++++++++++++++ etc/api_schemas/7.14/7.14.query.json | 323 ++++++++++++++ etc/api_schemas/7.14/7.14.threat_match.json | 408 ++++++++++++++++++ etc/api_schemas/7.14/7.14.threshold.json | 367 ++++++++++++++++ etc/api_schemas/master/master.base.json | 9 + etc/api_schemas/master/master.eql.json | 15 + .../master/master.machine_learning.json | 24 +- etc/api_schemas/master/master.query.json | 12 + .../master/master.threat_match.json | 15 + etc/api_schemas/master/master.threshold.json | 12 + requirements.txt | 5 +- 15 files changed, 2166 insertions(+), 5 deletions(-) create mode 100644 etc/api_schemas/7.14/7.14.base.json create mode 100644 etc/api_schemas/7.14/7.14.eql.json create mode 100644 etc/api_schemas/7.14/7.14.machine_learning.json create mode 100644 etc/api_schemas/7.14/7.14.query.json create mode 100644 etc/api_schemas/7.14/7.14.threat_match.json create mode 100644 etc/api_schemas/7.14/7.14.threshold.json diff --git a/detection_rules/mixins.py b/detection_rules/mixins.py index fc785e633..fbaa3d3fa 100644 --- a/detection_rules/mixins.py +++ b/detection_rules/mixins.py @@ -7,7 +7,9 @@ from typing import TypeVar, Type import marshmallow_dataclass +import marshmallow_dataclass.union_field import marshmallow_jsonschema +import marshmallow_union from marshmallow import Schema from .utils import cached @@ -42,7 +44,10 @@ def patch_jsonschema(obj: dict) -> dict: child.pop("title", None) - if isinstance(child["type"], list): + if "anyOf" in child: + child["anyOf"] = [dive(c) for c in child["anyOf"]] + + elif isinstance(child["type"], list): if 'null' in child["type"]: child["type"] = [t for t in child["type"] if t != 'null'] @@ -86,7 +91,7 @@ class MarshmallowDataclassMixin: @cached def jsonschema(cls): """Get the jsonschema representation for this class.""" - jsonschema = marshmallow_jsonschema.JSONSchema().dump(cls.__schema()) + jsonschema = PatchedJSONSchema().dump(cls.__schema()) jsonschema = patch_jsonschema(jsonschema) return jsonschema @@ -105,3 +110,20 @@ class MarshmallowDataclassMixin: serialized = _strip_none_from_dict(serialized) return serialized + + +class PatchedJSONSchema(marshmallow_jsonschema.JSONSchema): + + # Patch marshmallow-jsonschema to support marshmallow-dataclass[union] + def _get_schema_for_field(self, obj, field): + """Patch marshmallow_jsonschema.base.JSONSchema to support marshmallow-dataclass[union].""" + if isinstance(field, marshmallow_dataclass.union_field.Union): + # convert to marshmallow_union.Union + field = marshmallow_union.Union([subfield for _, subfield in field.union_fields], + metadata=field.metadata, + required=field.required, name=field.name, + parent=field.parent, root=field.root, error_messages=field.error_messages, + default_error_messages=field.default_error_messages, default=field.default, + allow_none=field.allow_none) + + return super()._get_schema_for_field(obj, field) diff --git a/detection_rules/rule.py b/detection_rules/rule.py index 2aad2b49a..d26624e9b 100644 --- a/detection_rules/rule.py +++ b/detection_rules/rule.py @@ -34,6 +34,7 @@ class RuleMeta(MarshmallowDataclassMixin): # Optional fields comments: Optional[str] + integration: Optional[str] maturity: Optional[definitions.Maturity] min_stack_version: Optional[definitions.SemVer] os_type_list: Optional[List[definitions.OSType]] diff --git a/etc/api_schemas/7.14/7.14.base.json b/etc/api_schemas/7.14/7.14.base.json new file mode 100644 index 000000000..2d1b94493 --- /dev/null +++ b/etc/api_schemas/7.14/7.14.base.json @@ -0,0 +1,310 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "additionalProperties": false, + "properties": { + "actions": { + "items": { + "type": "string" + }, + "type": "array" + }, + "author": { + "items": { + "type": "string" + }, + "type": "array" + }, + "building_block_type": { + "type": "string" + }, + "description": { + "type": "string" + }, + "enabled": { + "type": "boolean" + }, + "exceptions_list": { + "items": { + "type": "string" + }, + "type": "array" + }, + "false_positives": { + "items": { + "type": "string" + }, + "type": "array" + }, + "filters": { + "items": { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + "type": "array" + }, + "from": { + "type": "string" + }, + "interval": { + "description": "Interval", + "pattern": "\\d+[mshd]", + "type": "string" + }, + "license": { + "type": "string" + }, + "max_signals": { + "description": "MaxSignals", + "format": "integer", + "minimum": 1, + "type": "number" + }, + "meta": { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + "name": { + "type": "string" + }, + "note": { + "description": "MarkdownField", + "type": "string" + }, + "references": { + "items": { + "type": "string" + }, + "type": "array" + }, + "risk_score": { + "description": "MaxSignals", + "format": "integer", + "maximum": 100, + "minimum": 1, + "type": "number" + }, + "risk_score_mapping": { + "items": { + "additionalProperties": false, + "properties": { + "field": { + "type": "string" + }, + "operator": { + "enum": [ + "equals" + ], + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "field" + ], + "type": "object" + }, + "type": "array" + }, + "rule_id": { + "description": "UUIDString", + "pattern": "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", + "type": "string" + }, + "rule_name_override": { + "type": "string" + }, + "severity": { + "enum": [ + "low", + "medium", + "high", + "critical" + ], + "enumNames": [], + "type": "string" + }, + "severity_mapping": { + "items": { + "additionalProperties": false, + "properties": { + "field": { + "type": "string" + }, + "operator": { + "enum": [ + "equals" + ], + "type": "string" + }, + "severity": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "field" + ], + "type": "object" + }, + "type": "array" + }, + "tags": { + "items": { + "type": "string" + }, + "type": "array" + }, + "threat": { + "items": { + "additionalProperties": false, + "properties": { + "framework": { + "enum": [ + "MITRE ATT&CK" + ], + "type": "string" + }, + "tactic": { + "additionalProperties": false, + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "reference": { + "description": "TacticURL", + "pattern": "https://attack.mitre.org/tactics/TA[0-9]+/", + "type": "string" + } + }, + "required": [ + "id", + "name", + "reference" + ], + "type": "object" + }, + "technique": { + "items": { + "additionalProperties": false, + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "reference": { + "description": "TechniqueURL", + "pattern": "https://attack.mitre.org/techniques/T[0-9]+/", + "type": "string" + }, + "subtechnique": { + "items": { + "additionalProperties": false, + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "reference": { + "description": "SubTechniqueURL", + "pattern": "https://attack.mitre.org/techniques/T[0-9]+/[0-9]+/", + "type": "string" + } + }, + "required": [ + "id", + "name", + "reference" + ], + "type": "object" + }, + "type": "array" + } + }, + "required": [ + "id", + "name", + "reference" + ], + "type": "object" + }, + "type": "array" + } + }, + "required": [ + "framework", + "tactic" + ], + "type": "object" + }, + "type": "array" + }, + "throttle": { + "type": "string" + }, + "timeline_id": { + "description": "TimelineTemplateId", + "enum": [ + "db366523-f1c6-4c1f-8731-6ce5ed9e5717", + "91832785-286d-4ebe-b884-1a208d111a70", + "76e52245-7519-4251-91ab-262fb1a1728c", + "495ad7a7-316e-4544-8a0f-9c098daee76e" + ], + "enumNames": [], + "type": "string" + }, + "timeline_title": { + "description": "TimelineTemplateTitle", + "enum": [ + "Generic Endpoint Timeline", + "Generic Network Timeline", + "Generic Process Timeline", + "Generic Threat Match Timeline" + ], + "enumNames": [], + "type": "string" + }, + "timestamp_override": { + "type": "string" + }, + "to": { + "type": "string" + }, + "type": { + "enum": [ + "query", + "saved_query", + "machine_learning", + "eql", + "threshold", + "threat_match" + ], + "enumNames": [], + "type": "string" + } + }, + "required": [ + "author", + "description", + "name", + "risk_score", + "rule_id", + "severity", + "type" + ], + "type": "object" +} \ No newline at end of file diff --git a/etc/api_schemas/7.14/7.14.eql.json b/etc/api_schemas/7.14/7.14.eql.json new file mode 100644 index 000000000..4a37a0d1a --- /dev/null +++ b/etc/api_schemas/7.14/7.14.eql.json @@ -0,0 +1,321 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "additionalProperties": false, + "properties": { + "actions": { + "items": { + "type": "string" + }, + "type": "array" + }, + "author": { + "items": { + "type": "string" + }, + "type": "array" + }, + "building_block_type": { + "type": "string" + }, + "description": { + "type": "string" + }, + "enabled": { + "type": "boolean" + }, + "exceptions_list": { + "items": { + "type": "string" + }, + "type": "array" + }, + "false_positives": { + "items": { + "type": "string" + }, + "type": "array" + }, + "filters": { + "items": { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + "type": "array" + }, + "from": { + "type": "string" + }, + "index": { + "items": { + "type": "string" + }, + "type": "array" + }, + "interval": { + "description": "Interval", + "pattern": "\\d+[mshd]", + "type": "string" + }, + "language": { + "enum": [ + "eql" + ], + "type": "string" + }, + "license": { + "type": "string" + }, + "max_signals": { + "description": "MaxSignals", + "format": "integer", + "minimum": 1, + "type": "number" + }, + "meta": { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + "name": { + "type": "string" + }, + "note": { + "description": "MarkdownField", + "type": "string" + }, + "query": { + "type": "string" + }, + "references": { + "items": { + "type": "string" + }, + "type": "array" + }, + "risk_score": { + "description": "MaxSignals", + "format": "integer", + "maximum": 100, + "minimum": 1, + "type": "number" + }, + "risk_score_mapping": { + "items": { + "additionalProperties": false, + "properties": { + "field": { + "type": "string" + }, + "operator": { + "enum": [ + "equals" + ], + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "field" + ], + "type": "object" + }, + "type": "array" + }, + "rule_id": { + "description": "UUIDString", + "pattern": "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", + "type": "string" + }, + "rule_name_override": { + "type": "string" + }, + "severity": { + "enum": [ + "low", + "medium", + "high", + "critical" + ], + "enumNames": [], + "type": "string" + }, + "severity_mapping": { + "items": { + "additionalProperties": false, + "properties": { + "field": { + "type": "string" + }, + "operator": { + "enum": [ + "equals" + ], + "type": "string" + }, + "severity": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "field" + ], + "type": "object" + }, + "type": "array" + }, + "tags": { + "items": { + "type": "string" + }, + "type": "array" + }, + "threat": { + "items": { + "additionalProperties": false, + "properties": { + "framework": { + "enum": [ + "MITRE ATT&CK" + ], + "type": "string" + }, + "tactic": { + "additionalProperties": false, + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "reference": { + "description": "TacticURL", + "pattern": "https://attack.mitre.org/tactics/TA[0-9]+/", + "type": "string" + } + }, + "required": [ + "id", + "name", + "reference" + ], + "type": "object" + }, + "technique": { + "items": { + "additionalProperties": false, + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "reference": { + "description": "TechniqueURL", + "pattern": "https://attack.mitre.org/techniques/T[0-9]+/", + "type": "string" + }, + "subtechnique": { + "items": { + "additionalProperties": false, + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "reference": { + "description": "SubTechniqueURL", + "pattern": "https://attack.mitre.org/techniques/T[0-9]+/[0-9]+/", + "type": "string" + } + }, + "required": [ + "id", + "name", + "reference" + ], + "type": "object" + }, + "type": "array" + } + }, + "required": [ + "id", + "name", + "reference" + ], + "type": "object" + }, + "type": "array" + } + }, + "required": [ + "framework", + "tactic" + ], + "type": "object" + }, + "type": "array" + }, + "throttle": { + "type": "string" + }, + "timeline_id": { + "description": "TimelineTemplateId", + "enum": [ + "db366523-f1c6-4c1f-8731-6ce5ed9e5717", + "91832785-286d-4ebe-b884-1a208d111a70", + "76e52245-7519-4251-91ab-262fb1a1728c", + "495ad7a7-316e-4544-8a0f-9c098daee76e" + ], + "enumNames": [], + "type": "string" + }, + "timeline_title": { + "description": "TimelineTemplateTitle", + "enum": [ + "Generic Endpoint Timeline", + "Generic Network Timeline", + "Generic Process Timeline", + "Generic Threat Match Timeline" + ], + "enumNames": [], + "type": "string" + }, + "timestamp_override": { + "type": "string" + }, + "to": { + "type": "string" + }, + "type": { + "enum": [ + "eql" + ], + "type": "string" + } + }, + "required": [ + "author", + "description", + "language", + "name", + "query", + "risk_score", + "rule_id", + "severity", + "type" + ], + "type": "object" +} diff --git a/etc/api_schemas/7.14/7.14.machine_learning.json b/etc/api_schemas/7.14/7.14.machine_learning.json new file mode 100644 index 000000000..c8683bfc9 --- /dev/null +++ b/etc/api_schemas/7.14/7.14.machine_learning.json @@ -0,0 +1,323 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "additionalProperties": false, + "properties": { + "actions": { + "items": { + "type": "string" + }, + "type": "array" + }, + "anomaly_threshold": { + "format": "integer", + "type": "number" + }, + "author": { + "items": { + "type": "string" + }, + "type": "array" + }, + "building_block_type": { + "type": "string" + }, + "description": { + "type": "string" + }, + "enabled": { + "type": "boolean" + }, + "exceptions_list": { + "items": { + "type": "string" + }, + "type": "array" + }, + "false_positives": { + "items": { + "type": "string" + }, + "type": "array" + }, + "filters": { + "items": { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + "type": "array" + }, + "from": { + "type": "string" + }, + "interval": { + "description": "Interval", + "pattern": "\\d+[mshd]", + "type": "string" + }, + "license": { + "type": "string" + }, + "machine_learning_job_id": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ] + }, + "max_signals": { + "description": "MaxSignals", + "format": "integer", + "minimum": 1, + "type": "number" + }, + "meta": { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + "name": { + "type": "string" + }, + "note": { + "description": "MarkdownField", + "type": "string" + }, + "references": { + "items": { + "type": "string" + }, + "type": "array" + }, + "risk_score": { + "description": "MaxSignals", + "format": "integer", + "maximum": 100, + "minimum": 1, + "type": "number" + }, + "risk_score_mapping": { + "items": { + "additionalProperties": false, + "properties": { + "field": { + "type": "string" + }, + "operator": { + "enum": [ + "equals" + ], + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "field" + ], + "type": "object" + }, + "type": "array" + }, + "rule_id": { + "description": "UUIDString", + "pattern": "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", + "type": "string" + }, + "rule_name_override": { + "type": "string" + }, + "severity": { + "enum": [ + "low", + "medium", + "high", + "critical" + ], + "enumNames": [], + "type": "string" + }, + "severity_mapping": { + "items": { + "additionalProperties": false, + "properties": { + "field": { + "type": "string" + }, + "operator": { + "enum": [ + "equals" + ], + "type": "string" + }, + "severity": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "field" + ], + "type": "object" + }, + "type": "array" + }, + "tags": { + "items": { + "type": "string" + }, + "type": "array" + }, + "threat": { + "items": { + "additionalProperties": false, + "properties": { + "framework": { + "enum": [ + "MITRE ATT&CK" + ], + "type": "string" + }, + "tactic": { + "additionalProperties": false, + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "reference": { + "description": "TacticURL", + "pattern": "https://attack.mitre.org/tactics/TA[0-9]+/", + "type": "string" + } + }, + "required": [ + "id", + "name", + "reference" + ], + "type": "object" + }, + "technique": { + "items": { + "additionalProperties": false, + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "reference": { + "description": "TechniqueURL", + "pattern": "https://attack.mitre.org/techniques/T[0-9]+/", + "type": "string" + }, + "subtechnique": { + "items": { + "additionalProperties": false, + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "reference": { + "description": "SubTechniqueURL", + "pattern": "https://attack.mitre.org/techniques/T[0-9]+/[0-9]+/", + "type": "string" + } + }, + "required": [ + "id", + "name", + "reference" + ], + "type": "object" + }, + "type": "array" + } + }, + "required": [ + "id", + "name", + "reference" + ], + "type": "object" + }, + "type": "array" + } + }, + "required": [ + "framework", + "tactic" + ], + "type": "object" + }, + "type": "array" + }, + "throttle": { + "type": "string" + }, + "timeline_id": { + "description": "TimelineTemplateId", + "enum": [ + "db366523-f1c6-4c1f-8731-6ce5ed9e5717", + "91832785-286d-4ebe-b884-1a208d111a70", + "76e52245-7519-4251-91ab-262fb1a1728c", + "495ad7a7-316e-4544-8a0f-9c098daee76e" + ], + "enumNames": [], + "type": "string" + }, + "timeline_title": { + "description": "TimelineTemplateTitle", + "enum": [ + "Generic Endpoint Timeline", + "Generic Network Timeline", + "Generic Process Timeline", + "Generic Threat Match Timeline" + ], + "enumNames": [], + "type": "string" + }, + "timestamp_override": { + "type": "string" + }, + "to": { + "type": "string" + }, + "type": { + "enum": [ + "machine_learning" + ], + "type": "string" + } + }, + "required": [ + "anomaly_threshold", + "author", + "description", + "machine_learning_job_id", + "name", + "risk_score", + "rule_id", + "severity", + "type" + ], + "type": "object" +} \ No newline at end of file diff --git a/etc/api_schemas/7.14/7.14.query.json b/etc/api_schemas/7.14/7.14.query.json new file mode 100644 index 000000000..3d1c02922 --- /dev/null +++ b/etc/api_schemas/7.14/7.14.query.json @@ -0,0 +1,323 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "additionalProperties": false, + "properties": { + "actions": { + "items": { + "type": "string" + }, + "type": "array" + }, + "author": { + "items": { + "type": "string" + }, + "type": "array" + }, + "building_block_type": { + "type": "string" + }, + "description": { + "type": "string" + }, + "enabled": { + "type": "boolean" + }, + "exceptions_list": { + "items": { + "type": "string" + }, + "type": "array" + }, + "false_positives": { + "items": { + "type": "string" + }, + "type": "array" + }, + "filters": { + "items": { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + "type": "array" + }, + "from": { + "type": "string" + }, + "index": { + "items": { + "type": "string" + }, + "type": "array" + }, + "interval": { + "description": "Interval", + "pattern": "\\d+[mshd]", + "type": "string" + }, + "language": { + "enum": [ + "kuery", + "lucene" + ], + "enumNames": [], + "type": "string" + }, + "license": { + "type": "string" + }, + "max_signals": { + "description": "MaxSignals", + "format": "integer", + "minimum": 1, + "type": "number" + }, + "meta": { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + "name": { + "type": "string" + }, + "note": { + "description": "MarkdownField", + "type": "string" + }, + "query": { + "type": "string" + }, + "references": { + "items": { + "type": "string" + }, + "type": "array" + }, + "risk_score": { + "description": "MaxSignals", + "format": "integer", + "maximum": 100, + "minimum": 1, + "type": "number" + }, + "risk_score_mapping": { + "items": { + "additionalProperties": false, + "properties": { + "field": { + "type": "string" + }, + "operator": { + "enum": [ + "equals" + ], + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "field" + ], + "type": "object" + }, + "type": "array" + }, + "rule_id": { + "description": "UUIDString", + "pattern": "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", + "type": "string" + }, + "rule_name_override": { + "type": "string" + }, + "severity": { + "enum": [ + "low", + "medium", + "high", + "critical" + ], + "enumNames": [], + "type": "string" + }, + "severity_mapping": { + "items": { + "additionalProperties": false, + "properties": { + "field": { + "type": "string" + }, + "operator": { + "enum": [ + "equals" + ], + "type": "string" + }, + "severity": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "field" + ], + "type": "object" + }, + "type": "array" + }, + "tags": { + "items": { + "type": "string" + }, + "type": "array" + }, + "threat": { + "items": { + "additionalProperties": false, + "properties": { + "framework": { + "enum": [ + "MITRE ATT&CK" + ], + "type": "string" + }, + "tactic": { + "additionalProperties": false, + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "reference": { + "description": "TacticURL", + "pattern": "https://attack.mitre.org/tactics/TA[0-9]+/", + "type": "string" + } + }, + "required": [ + "id", + "name", + "reference" + ], + "type": "object" + }, + "technique": { + "items": { + "additionalProperties": false, + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "reference": { + "description": "TechniqueURL", + "pattern": "https://attack.mitre.org/techniques/T[0-9]+/", + "type": "string" + }, + "subtechnique": { + "items": { + "additionalProperties": false, + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "reference": { + "description": "SubTechniqueURL", + "pattern": "https://attack.mitre.org/techniques/T[0-9]+/[0-9]+/", + "type": "string" + } + }, + "required": [ + "id", + "name", + "reference" + ], + "type": "object" + }, + "type": "array" + } + }, + "required": [ + "id", + "name", + "reference" + ], + "type": "object" + }, + "type": "array" + } + }, + "required": [ + "framework", + "tactic" + ], + "type": "object" + }, + "type": "array" + }, + "throttle": { + "type": "string" + }, + "timeline_id": { + "description": "TimelineTemplateId", + "enum": [ + "db366523-f1c6-4c1f-8731-6ce5ed9e5717", + "91832785-286d-4ebe-b884-1a208d111a70", + "76e52245-7519-4251-91ab-262fb1a1728c", + "495ad7a7-316e-4544-8a0f-9c098daee76e" + ], + "enumNames": [], + "type": "string" + }, + "timeline_title": { + "description": "TimelineTemplateTitle", + "enum": [ + "Generic Endpoint Timeline", + "Generic Network Timeline", + "Generic Process Timeline", + "Generic Threat Match Timeline" + ], + "enumNames": [], + "type": "string" + }, + "timestamp_override": { + "type": "string" + }, + "to": { + "type": "string" + }, + "type": { + "enum": [ + "query" + ], + "type": "string" + } + }, + "required": [ + "author", + "description", + "language", + "name", + "query", + "risk_score", + "rule_id", + "severity", + "type" + ], + "type": "object" +} \ No newline at end of file diff --git a/etc/api_schemas/7.14/7.14.threat_match.json b/etc/api_schemas/7.14/7.14.threat_match.json new file mode 100644 index 000000000..222c75782 --- /dev/null +++ b/etc/api_schemas/7.14/7.14.threat_match.json @@ -0,0 +1,408 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "additionalProperties": false, + "properties": { + "actions": { + "items": { + "type": "string" + }, + "type": "array" + }, + "author": { + "items": { + "type": "string" + }, + "type": "array" + }, + "building_block_type": { + "type": "string" + }, + "concurrent_searches": { + "description": "PositiveInteger", + "format": "integer", + "minimum": 1, + "type": "number" + }, + "description": { + "type": "string" + }, + "enabled": { + "type": "boolean" + }, + "exceptions_list": { + "items": { + "type": "string" + }, + "type": "array" + }, + "false_positives": { + "items": { + "type": "string" + }, + "type": "array" + }, + "filters": { + "items": { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + "type": "array" + }, + "from": { + "type": "string" + }, + "index": { + "items": { + "type": "string" + }, + "type": "array" + }, + "interval": { + "description": "Interval", + "pattern": "\\d+[mshd]", + "type": "string" + }, + "items_per_search": { + "description": "PositiveInteger", + "format": "integer", + "minimum": 1, + "type": "number" + }, + "language": { + "enum": [ + "kuery", + "lucene" + ], + "enumNames": [], + "type": "string" + }, + "license": { + "type": "string" + }, + "max_signals": { + "description": "MaxSignals", + "format": "integer", + "minimum": 1, + "type": "number" + }, + "meta": { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + "name": { + "type": "string" + }, + "note": { + "description": "MarkdownField", + "type": "string" + }, + "query": { + "type": "string" + }, + "references": { + "items": { + "type": "string" + }, + "type": "array" + }, + "risk_score": { + "description": "MaxSignals", + "format": "integer", + "maximum": 100, + "minimum": 1, + "type": "number" + }, + "risk_score_mapping": { + "items": { + "additionalProperties": false, + "properties": { + "field": { + "type": "string" + }, + "operator": { + "enum": [ + "equals" + ], + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "field" + ], + "type": "object" + }, + "type": "array" + }, + "rule_id": { + "description": "UUIDString", + "pattern": "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", + "type": "string" + }, + "rule_name_override": { + "type": "string" + }, + "severity": { + "enum": [ + "low", + "medium", + "high", + "critical" + ], + "enumNames": [], + "type": "string" + }, + "severity_mapping": { + "items": { + "additionalProperties": false, + "properties": { + "field": { + "type": "string" + }, + "operator": { + "enum": [ + "equals" + ], + "type": "string" + }, + "severity": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "field" + ], + "type": "object" + }, + "type": "array" + }, + "tags": { + "items": { + "type": "string" + }, + "type": "array" + }, + "threat": { + "items": { + "additionalProperties": false, + "properties": { + "framework": { + "enum": [ + "MITRE ATT&CK" + ], + "type": "string" + }, + "tactic": { + "additionalProperties": false, + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "reference": { + "description": "TacticURL", + "pattern": "https://attack.mitre.org/tactics/TA[0-9]+/", + "type": "string" + } + }, + "required": [ + "id", + "name", + "reference" + ], + "type": "object" + }, + "technique": { + "items": { + "additionalProperties": false, + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "reference": { + "description": "TechniqueURL", + "pattern": "https://attack.mitre.org/techniques/T[0-9]+/", + "type": "string" + }, + "subtechnique": { + "items": { + "additionalProperties": false, + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "reference": { + "description": "SubTechniqueURL", + "pattern": "https://attack.mitre.org/techniques/T[0-9]+/[0-9]+/", + "type": "string" + } + }, + "required": [ + "id", + "name", + "reference" + ], + "type": "object" + }, + "type": "array" + } + }, + "required": [ + "id", + "name", + "reference" + ], + "type": "object" + }, + "type": "array" + } + }, + "required": [ + "framework", + "tactic" + ], + "type": "object" + }, + "type": "array" + }, + "threat_filters": { + "items": { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + "type": "array" + }, + "threat_index": { + "items": { + "type": "string" + }, + "type": "array" + }, + "threat_indicator_path": { + "type": "string" + }, + "threat_language": { + "enum": [ + "kuery", + "lucene" + ], + "enumNames": [], + "type": "string" + }, + "threat_mapping": { + "items": { + "additionalProperties": false, + "properties": { + "entries": { + "items": { + "additionalProperties": false, + "properties": { + "field": { + "description": "NonEmptyStr", + "minLength": 1, + "type": "string" + }, + "type": { + "enum": [ + "mapping" + ], + "type": "string" + }, + "value": { + "description": "NonEmptyStr", + "minLength": 1, + "type": "string" + } + }, + "required": [ + "field", + "type", + "value" + ], + "type": "object" + }, + "type": "array" + } + }, + "required": [ + "entries" + ], + "type": "object" + }, + "type": "array" + }, + "threat_query": { + "type": "string" + }, + "throttle": { + "type": "string" + }, + "timeline_id": { + "description": "TimelineTemplateId", + "enum": [ + "db366523-f1c6-4c1f-8731-6ce5ed9e5717", + "91832785-286d-4ebe-b884-1a208d111a70", + "76e52245-7519-4251-91ab-262fb1a1728c", + "495ad7a7-316e-4544-8a0f-9c098daee76e" + ], + "enumNames": [], + "type": "string" + }, + "timeline_title": { + "description": "TimelineTemplateTitle", + "enum": [ + "Generic Endpoint Timeline", + "Generic Network Timeline", + "Generic Process Timeline", + "Generic Threat Match Timeline" + ], + "enumNames": [], + "type": "string" + }, + "timestamp_override": { + "type": "string" + }, + "to": { + "type": "string" + }, + "type": { + "enum": [ + "threat_match" + ], + "type": "string" + } + }, + "required": [ + "author", + "description", + "language", + "name", + "query", + "risk_score", + "rule_id", + "severity", + "threat_index", + "threat_mapping", + "type" + ], + "type": "object" +} \ No newline at end of file diff --git a/etc/api_schemas/7.14/7.14.threshold.json b/etc/api_schemas/7.14/7.14.threshold.json new file mode 100644 index 000000000..e07151206 --- /dev/null +++ b/etc/api_schemas/7.14/7.14.threshold.json @@ -0,0 +1,367 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "additionalProperties": false, + "properties": { + "actions": { + "items": { + "type": "string" + }, + "type": "array" + }, + "author": { + "items": { + "type": "string" + }, + "type": "array" + }, + "building_block_type": { + "type": "string" + }, + "description": { + "type": "string" + }, + "enabled": { + "type": "boolean" + }, + "exceptions_list": { + "items": { + "type": "string" + }, + "type": "array" + }, + "false_positives": { + "items": { + "type": "string" + }, + "type": "array" + }, + "filters": { + "items": { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + "type": "array" + }, + "from": { + "type": "string" + }, + "index": { + "items": { + "type": "string" + }, + "type": "array" + }, + "interval": { + "description": "Interval", + "pattern": "\\d+[mshd]", + "type": "string" + }, + "language": { + "enum": [ + "kuery", + "lucene" + ], + "enumNames": [], + "type": "string" + }, + "license": { + "type": "string" + }, + "max_signals": { + "description": "MaxSignals", + "format": "integer", + "minimum": 1, + "type": "number" + }, + "meta": { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + "name": { + "type": "string" + }, + "note": { + "description": "MarkdownField", + "type": "string" + }, + "query": { + "type": "string" + }, + "references": { + "items": { + "type": "string" + }, + "type": "array" + }, + "risk_score": { + "description": "MaxSignals", + "format": "integer", + "maximum": 100, + "minimum": 1, + "type": "number" + }, + "risk_score_mapping": { + "items": { + "additionalProperties": false, + "properties": { + "field": { + "type": "string" + }, + "operator": { + "enum": [ + "equals" + ], + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "field" + ], + "type": "object" + }, + "type": "array" + }, + "rule_id": { + "description": "UUIDString", + "pattern": "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", + "type": "string" + }, + "rule_name_override": { + "type": "string" + }, + "severity": { + "enum": [ + "low", + "medium", + "high", + "critical" + ], + "enumNames": [], + "type": "string" + }, + "severity_mapping": { + "items": { + "additionalProperties": false, + "properties": { + "field": { + "type": "string" + }, + "operator": { + "enum": [ + "equals" + ], + "type": "string" + }, + "severity": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "field" + ], + "type": "object" + }, + "type": "array" + }, + "tags": { + "items": { + "type": "string" + }, + "type": "array" + }, + "threat": { + "items": { + "additionalProperties": false, + "properties": { + "framework": { + "enum": [ + "MITRE ATT&CK" + ], + "type": "string" + }, + "tactic": { + "additionalProperties": false, + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "reference": { + "description": "TacticURL", + "pattern": "https://attack.mitre.org/tactics/TA[0-9]+/", + "type": "string" + } + }, + "required": [ + "id", + "name", + "reference" + ], + "type": "object" + }, + "technique": { + "items": { + "additionalProperties": false, + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "reference": { + "description": "TechniqueURL", + "pattern": "https://attack.mitre.org/techniques/T[0-9]+/", + "type": "string" + }, + "subtechnique": { + "items": { + "additionalProperties": false, + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "reference": { + "description": "SubTechniqueURL", + "pattern": "https://attack.mitre.org/techniques/T[0-9]+/[0-9]+/", + "type": "string" + } + }, + "required": [ + "id", + "name", + "reference" + ], + "type": "object" + }, + "type": "array" + } + }, + "required": [ + "id", + "name", + "reference" + ], + "type": "object" + }, + "type": "array" + } + }, + "required": [ + "framework", + "tactic" + ], + "type": "object" + }, + "type": "array" + }, + "threshold": { + "additionalProperties": false, + "properties": { + "cardinality": { + "additionalProperties": false, + "properties": { + "field": { + "type": "string" + }, + "value": { + "description": "ThresholdValue", + "format": "integer", + "minimum": 1, + "type": "number" + } + }, + "required": [ + "field", + "value" + ], + "type": "object" + }, + "field": { + "items": { + "description": "NonEmptyStr", + "minLength": 1, + "type": "string" + }, + "type": "array" + }, + "value": { + "description": "ThresholdValue", + "format": "integer", + "minimum": 1, + "type": "number" + } + }, + "required": [ + "field", + "value" + ], + "type": "object" + }, + "throttle": { + "type": "string" + }, + "timeline_id": { + "description": "TimelineTemplateId", + "enum": [ + "db366523-f1c6-4c1f-8731-6ce5ed9e5717", + "91832785-286d-4ebe-b884-1a208d111a70", + "76e52245-7519-4251-91ab-262fb1a1728c", + "495ad7a7-316e-4544-8a0f-9c098daee76e" + ], + "enumNames": [], + "type": "string" + }, + "timeline_title": { + "description": "TimelineTemplateTitle", + "enum": [ + "Generic Endpoint Timeline", + "Generic Network Timeline", + "Generic Process Timeline", + "Generic Threat Match Timeline" + ], + "enumNames": [], + "type": "string" + }, + "timestamp_override": { + "type": "string" + }, + "to": { + "type": "string" + }, + "type": { + "enum": [ + "threshold" + ], + "type": "string" + } + }, + "required": [ + "author", + "description", + "language", + "name", + "query", + "risk_score", + "rule_id", + "severity", + "threshold", + "type" + ], + "type": "object" +} \ No newline at end of file diff --git a/etc/api_schemas/master/master.base.json b/etc/api_schemas/master/master.base.json index 7750cc1ac..2d1b94493 100644 --- a/etc/api_schemas/master/master.base.json +++ b/etc/api_schemas/master/master.base.json @@ -95,6 +95,9 @@ "type": "string" }, "operator": { + "enum": [ + "equals" + ], "type": "string" }, "value": { @@ -134,6 +137,9 @@ "type": "string" }, "operator": { + "enum": [ + "equals" + ], "type": "string" }, "severity": { @@ -161,6 +167,9 @@ "additionalProperties": false, "properties": { "framework": { + "enum": [ + "MITRE ATT&CK" + ], "type": "string" }, "tactic": { diff --git a/etc/api_schemas/master/master.eql.json b/etc/api_schemas/master/master.eql.json index 381659f5e..3000f75db 100644 --- a/etc/api_schemas/master/master.eql.json +++ b/etc/api_schemas/master/master.eql.json @@ -59,6 +59,9 @@ "type": "string" }, "language": { + "enum": [ + "eql" + ], "type": "string" }, "license": { @@ -107,6 +110,9 @@ "type": "string" }, "operator": { + "enum": [ + "equals" + ], "type": "string" }, "value": { @@ -146,6 +152,9 @@ "type": "string" }, "operator": { + "enum": [ + "equals" + ], "type": "string" }, "severity": { @@ -173,6 +182,9 @@ "additionalProperties": false, "properties": { "framework": { + "enum": [ + "MITRE ATT&CK" + ], "type": "string" }, "tactic": { @@ -288,6 +300,9 @@ "type": "string" }, "type": { + "enum": [ + "eql" + ], "type": "string" } }, diff --git a/etc/api_schemas/master/master.machine_learning.json b/etc/api_schemas/master/master.machine_learning.json index 181f305aa..c8683bfc9 100644 --- a/etc/api_schemas/master/master.machine_learning.json +++ b/etc/api_schemas/master/master.machine_learning.json @@ -60,7 +60,17 @@ "type": "string" }, "machine_learning_job_id": { - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ] }, "max_signals": { "description": "MaxSignals", @@ -102,6 +112,9 @@ "type": "string" }, "operator": { + "enum": [ + "equals" + ], "type": "string" }, "value": { @@ -141,6 +154,9 @@ "type": "string" }, "operator": { + "enum": [ + "equals" + ], "type": "string" }, "severity": { @@ -168,6 +184,9 @@ "additionalProperties": false, "properties": { "framework": { + "enum": [ + "MITRE ATT&CK" + ], "type": "string" }, "tactic": { @@ -283,6 +302,9 @@ "type": "string" }, "type": { + "enum": [ + "machine_learning" + ], "type": "string" } }, diff --git a/etc/api_schemas/master/master.query.json b/etc/api_schemas/master/master.query.json index 9e47b2969..3d1c02922 100644 --- a/etc/api_schemas/master/master.query.json +++ b/etc/api_schemas/master/master.query.json @@ -112,6 +112,9 @@ "type": "string" }, "operator": { + "enum": [ + "equals" + ], "type": "string" }, "value": { @@ -151,6 +154,9 @@ "type": "string" }, "operator": { + "enum": [ + "equals" + ], "type": "string" }, "severity": { @@ -178,6 +184,9 @@ "additionalProperties": false, "properties": { "framework": { + "enum": [ + "MITRE ATT&CK" + ], "type": "string" }, "tactic": { @@ -293,6 +302,9 @@ "type": "string" }, "type": { + "enum": [ + "query" + ], "type": "string" } }, diff --git a/etc/api_schemas/master/master.threat_match.json b/etc/api_schemas/master/master.threat_match.json index 84a603c3d..222c75782 100644 --- a/etc/api_schemas/master/master.threat_match.json +++ b/etc/api_schemas/master/master.threat_match.json @@ -124,6 +124,9 @@ "type": "string" }, "operator": { + "enum": [ + "equals" + ], "type": "string" }, "value": { @@ -163,6 +166,9 @@ "type": "string" }, "operator": { + "enum": [ + "equals" + ], "type": "string" }, "severity": { @@ -190,6 +196,9 @@ "additionalProperties": false, "properties": { "framework": { + "enum": [ + "MITRE ATT&CK" + ], "type": "string" }, "tactic": { @@ -313,6 +322,9 @@ "type": "string" }, "type": { + "enum": [ + "mapping" + ], "type": "string" }, "value": { @@ -373,6 +385,9 @@ "type": "string" }, "type": { + "enum": [ + "threat_match" + ], "type": "string" } }, diff --git a/etc/api_schemas/master/master.threshold.json b/etc/api_schemas/master/master.threshold.json index 724d46cce..e07151206 100644 --- a/etc/api_schemas/master/master.threshold.json +++ b/etc/api_schemas/master/master.threshold.json @@ -112,6 +112,9 @@ "type": "string" }, "operator": { + "enum": [ + "equals" + ], "type": "string" }, "value": { @@ -151,6 +154,9 @@ "type": "string" }, "operator": { + "enum": [ + "equals" + ], "type": "string" }, "severity": { @@ -178,6 +184,9 @@ "additionalProperties": false, "properties": { "framework": { + "enum": [ + "MITRE ATT&CK" + ], "type": "string" }, "tactic": { @@ -336,6 +345,9 @@ "type": "string" }, "type": { + "enum": [ + "threshold" + ], "type": "string" } }, diff --git a/requirements.txt b/requirements.txt index 64138b25a..9292b8600 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,7 @@ eql==0.9.9 elasticsearch~=7.9 XlsxWriter~=1.3.6 marshmallow~=3.10.0 -marshmallow-dataclass[union]~=8.3.1 +marshmallow-dataclass[union]~=8.4 # test deps pyflakes==2.2.0 @@ -17,4 +17,5 @@ flake8==3.8.1 pep8-naming==0.7.0 pytest>=3.6 jsonschema==3.2.0 -marshmallow-jsonschema==0.11.1 +marshmallow-jsonschema~=0.12.0 +marshmallow-union~=0.1.15 # needed for marshmallow-jsonschem \ No newline at end of file