diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index 047a3f344..34e48be75 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -55,7 +55,8 @@ jobs: strategy: max-parallel: 1 matrix: - target_branch: [7.13, 7.14, 7.15, 7.16] + # 7.17 was intentionally skipped because it was added late and was bug fix only + target_branch: [7.13, 7.14, 7.15, 7.16, 8.0] steps: - name: Checkout repo diff --git a/.github/workflows/lock-versions.yml b/.github/workflows/lock-versions.yml index f6c20cc32..a03539ff2 100644 --- a/.github/workflows/lock-versions.yml +++ b/.github/workflows/lock-versions.yml @@ -5,7 +5,8 @@ on: branches: description: 'List of branches to lock versions (ordered, comma separated)' required: true - default: '7.13,7.14,7.15,7.16' + # 7.17 was intentionally skipped because it was added late and was bug fix only + default: '7.13,7.14,7.15,7.16,8.0' jobs: pr: diff --git a/detection_rules/rule_validators.py b/detection_rules/rule_validators.py index c1771c945..49031d2ad 100644 --- a/detection_rules/rule_validators.py +++ b/detection_rules/rule_validators.py @@ -5,7 +5,7 @@ """Validation logic for rules containing queries.""" from functools import cached_property -from typing import List +from typing import List, Optional, Union import eql @@ -104,3 +104,10 @@ class EQLValidator(QueryValidator): except Exception: print(err_trailer) raise + + +def extract_error_field(exc: Union[eql.EqlParseError, kql.KqlParseError]) -> Optional[str]: + line = exc.source.splitlines()[exc.line] + start = exc.column + stop = start + len(exc.caret.strip()) + return line[start:stop] diff --git a/detection_rules/schemas/__init__.py b/detection_rules/schemas/__init__.py index 20f4c1f99..6f8283e83 100644 --- a/detection_rules/schemas/__init__.py +++ b/detection_rules/schemas/__init__.py @@ -178,6 +178,12 @@ def migrate_to_7_16(version: Version, api_contents: dict) -> dict: return strip_additional_properties(version, api_contents) +@migrate("8.0") +def migrate_to_8_0(version: Version, api_contents: dict) -> dict: + """Default migration for 8.0.""" + return strip_additional_properties(version, api_contents) + + def downgrade(api_contents: dict, target_version: str, current_version: Optional[str] = None) -> dict: """Downgrade a rule to a target stack version.""" from ..packaging import current_stack_version diff --git a/detection_rules/semver.py b/detection_rules/semver.py index b032bff06..85a83ca92 100644 --- a/detection_rules/semver.py +++ b/detection_rules/semver.py @@ -10,17 +10,22 @@ from typing import Iterable, Union class Version(tuple): - def __new__(cls, version: Union[Iterable, str]) -> 'Version': - if not isinstance(version, (int, list, tuple)): - version = tuple(int(a) if a.isdigit() else a for a in re.split(r'[.-]', version)) - return version if isinstance(version, int) else tuple.__new__(cls, version) + def __new__(cls, version: Union[str, Iterable]) -> 'Version': + if isinstance(version, (int, list, tuple)): + version_class = tuple.__new__(cls, version) + else: + version_tuple = tuple(int(a) if a.isdigit() else a for a in re.split(r'[.-]', version)) + version_class = tuple.__new__(cls, version_tuple) - def bump(self): - """Increment the version.""" - versions = list(self) - versions[-1] += 1 - return Version(versions) + return version_class def __str__(self): """Convert back to a string.""" - return ".".join(str(dig) for dig in self) + recovered_str = str(self[0]) + for additional in self[1:]: + if isinstance(additional, str): + recovered_str += "-" + additional + else: + recovered_str += "." + str(additional) + + return recovered_str diff --git a/etc/api_schemas/8.0/8.0.base.json b/etc/api_schemas/8.0/8.0.base.json new file mode 100644 index 000000000..1f5e717e2 --- /dev/null +++ b/etc/api_schemas/8.0/8.0.base.json @@ -0,0 +1,334 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "additionalProperties": false, + "properties": { + "actions": { + "items": { + "type": [ + "string", + "number", + "object", + "array", + "boolean" + ] + }, + "type": "array" + }, + "author": { + "items": { + "type": "string" + }, + "type": "array" + }, + "building_block_type": { + "type": "string" + }, + "description": { + "type": "string" + }, + "enabled": { + "type": "boolean" + }, + "exceptions_list": { + "items": { + "type": [ + "string", + "number", + "object", + "array", + "boolean" + ] + }, + "type": "array" + }, + "false_positives": { + "items": { + "type": "string" + }, + "type": "array" + }, + "filters": { + "items": { + "additionalProperties": { + "type": [ + "string", + "number", + "object", + "array", + "boolean" + ] + }, + "type": "object" + }, + "type": "array" + }, + "from": { + "type": "string" + }, + "interval": { + "description": "Interval", + "pattern": "^\\d+[mshd]$", + "type": "string" + }, + "license": { + "type": "string" + }, + "max_signals": { + "description": "MaxSignals", + "format": "integer", + "minimum": 1, + "type": "number" + }, + "meta": { + "additionalProperties": { + "type": [ + "string", + "number", + "object", + "array", + "boolean" + ] + }, + "type": "object" + }, + "name": { + "type": "string" + }, + "note": { + "description": "MarkdownField", + "type": "string" + }, + "references": { + "items": { + "type": "string" + }, + "type": "array" + }, + "risk_score": { + "description": "MaxSignals", + "format": "integer", + "maximum": 100, + "minimum": 1, + "type": "number" + }, + "risk_score_mapping": { + "items": { + "additionalProperties": false, + "properties": { + "field": { + "type": "string" + }, + "operator": { + "enum": [ + "equals" + ], + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "field" + ], + "type": "object" + }, + "type": "array" + }, + "rule_id": { + "description": "UUIDString", + "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", + "type": "string" + }, + "rule_name_override": { + "type": "string" + }, + "severity": { + "enum": [ + "low", + "medium", + "high", + "critical" + ], + "enumNames": [], + "type": "string" + }, + "severity_mapping": { + "items": { + "additionalProperties": false, + "properties": { + "field": { + "type": "string" + }, + "operator": { + "enum": [ + "equals" + ], + "type": "string" + }, + "severity": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "field" + ], + "type": "object" + }, + "type": "array" + }, + "tags": { + "items": { + "type": "string" + }, + "type": "array" + }, + "threat": { + "items": { + "additionalProperties": false, + "properties": { + "framework": { + "enum": [ + "MITRE ATT&CK" + ], + "type": "string" + }, + "tactic": { + "additionalProperties": false, + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "reference": { + "description": "TacticURL", + "pattern": "^https://attack.mitre.org/tactics/TA[0-9]+/$", + "type": "string" + } + }, + "required": [ + "id", + "name", + "reference" + ], + "type": "object" + }, + "technique": { + "items": { + "additionalProperties": false, + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "reference": { + "description": "TechniqueURL", + "pattern": "^https://attack.mitre.org/techniques/T[0-9]+/$", + "type": "string" + }, + "subtechnique": { + "items": { + "additionalProperties": false, + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "reference": { + "description": "SubTechniqueURL", + "pattern": "^https://attack.mitre.org/techniques/T[0-9]+/[0-9]+/$", + "type": "string" + } + }, + "required": [ + "id", + "name", + "reference" + ], + "type": "object" + }, + "type": "array" + } + }, + "required": [ + "id", + "name", + "reference" + ], + "type": "object" + }, + "type": "array" + } + }, + "required": [ + "framework", + "tactic" + ], + "type": "object" + }, + "type": "array" + }, + "throttle": { + "type": "string" + }, + "timeline_id": { + "description": "TimelineTemplateId", + "enum": [ + "db366523-f1c6-4c1f-8731-6ce5ed9e5717", + "91832785-286d-4ebe-b884-1a208d111a70", + "76e52245-7519-4251-91ab-262fb1a1728c", + "495ad7a7-316e-4544-8a0f-9c098daee76e" + ], + "enumNames": [], + "type": "string" + }, + "timeline_title": { + "description": "TimelineTemplateTitle", + "enum": [ + "Generic Endpoint Timeline", + "Generic Network Timeline", + "Generic Process Timeline", + "Generic Threat Match Timeline" + ], + "enumNames": [], + "type": "string" + }, + "timestamp_override": { + "type": "string" + }, + "to": { + "type": "string" + }, + "type": { + "enum": [ + "query", + "saved_query", + "machine_learning", + "eql", + "threshold", + "threat_match" + ], + "enumNames": [], + "type": "string" + } + }, + "required": [ + "author", + "description", + "name", + "risk_score", + "rule_id", + "severity", + "type" + ], + "type": "object" +} \ No newline at end of file diff --git a/etc/api_schemas/8.0/8.0.eql.json b/etc/api_schemas/8.0/8.0.eql.json new file mode 100644 index 000000000..103e625c9 --- /dev/null +++ b/etc/api_schemas/8.0/8.0.eql.json @@ -0,0 +1,345 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "additionalProperties": false, + "properties": { + "actions": { + "items": { + "type": [ + "string", + "number", + "object", + "array", + "boolean" + ] + }, + "type": "array" + }, + "author": { + "items": { + "type": "string" + }, + "type": "array" + }, + "building_block_type": { + "type": "string" + }, + "description": { + "type": "string" + }, + "enabled": { + "type": "boolean" + }, + "exceptions_list": { + "items": { + "type": [ + "string", + "number", + "object", + "array", + "boolean" + ] + }, + "type": "array" + }, + "false_positives": { + "items": { + "type": "string" + }, + "type": "array" + }, + "filters": { + "items": { + "additionalProperties": { + "type": [ + "string", + "number", + "object", + "array", + "boolean" + ] + }, + "type": "object" + }, + "type": "array" + }, + "from": { + "type": "string" + }, + "index": { + "items": { + "type": "string" + }, + "type": "array" + }, + "interval": { + "description": "Interval", + "pattern": "^\\d+[mshd]$", + "type": "string" + }, + "language": { + "enum": [ + "eql" + ], + "type": "string" + }, + "license": { + "type": "string" + }, + "max_signals": { + "description": "MaxSignals", + "format": "integer", + "minimum": 1, + "type": "number" + }, + "meta": { + "additionalProperties": { + "type": [ + "string", + "number", + "object", + "array", + "boolean" + ] + }, + "type": "object" + }, + "name": { + "type": "string" + }, + "note": { + "description": "MarkdownField", + "type": "string" + }, + "query": { + "type": "string" + }, + "references": { + "items": { + "type": "string" + }, + "type": "array" + }, + "risk_score": { + "description": "MaxSignals", + "format": "integer", + "maximum": 100, + "minimum": 1, + "type": "number" + }, + "risk_score_mapping": { + "items": { + "additionalProperties": false, + "properties": { + "field": { + "type": "string" + }, + "operator": { + "enum": [ + "equals" + ], + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "field" + ], + "type": "object" + }, + "type": "array" + }, + "rule_id": { + "description": "UUIDString", + "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", + "type": "string" + }, + "rule_name_override": { + "type": "string" + }, + "severity": { + "enum": [ + "low", + "medium", + "high", + "critical" + ], + "enumNames": [], + "type": "string" + }, + "severity_mapping": { + "items": { + "additionalProperties": false, + "properties": { + "field": { + "type": "string" + }, + "operator": { + "enum": [ + "equals" + ], + "type": "string" + }, + "severity": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "field" + ], + "type": "object" + }, + "type": "array" + }, + "tags": { + "items": { + "type": "string" + }, + "type": "array" + }, + "threat": { + "items": { + "additionalProperties": false, + "properties": { + "framework": { + "enum": [ + "MITRE ATT&CK" + ], + "type": "string" + }, + "tactic": { + "additionalProperties": false, + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "reference": { + "description": "TacticURL", + "pattern": "^https://attack.mitre.org/tactics/TA[0-9]+/$", + "type": "string" + } + }, + "required": [ + "id", + "name", + "reference" + ], + "type": "object" + }, + "technique": { + "items": { + "additionalProperties": false, + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "reference": { + "description": "TechniqueURL", + "pattern": "^https://attack.mitre.org/techniques/T[0-9]+/$", + "type": "string" + }, + "subtechnique": { + "items": { + "additionalProperties": false, + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "reference": { + "description": "SubTechniqueURL", + "pattern": "^https://attack.mitre.org/techniques/T[0-9]+/[0-9]+/$", + "type": "string" + } + }, + "required": [ + "id", + "name", + "reference" + ], + "type": "object" + }, + "type": "array" + } + }, + "required": [ + "id", + "name", + "reference" + ], + "type": "object" + }, + "type": "array" + } + }, + "required": [ + "framework", + "tactic" + ], + "type": "object" + }, + "type": "array" + }, + "throttle": { + "type": "string" + }, + "timeline_id": { + "description": "TimelineTemplateId", + "enum": [ + "db366523-f1c6-4c1f-8731-6ce5ed9e5717", + "91832785-286d-4ebe-b884-1a208d111a70", + "76e52245-7519-4251-91ab-262fb1a1728c", + "495ad7a7-316e-4544-8a0f-9c098daee76e" + ], + "enumNames": [], + "type": "string" + }, + "timeline_title": { + "description": "TimelineTemplateTitle", + "enum": [ + "Generic Endpoint Timeline", + "Generic Network Timeline", + "Generic Process Timeline", + "Generic Threat Match Timeline" + ], + "enumNames": [], + "type": "string" + }, + "timestamp_override": { + "type": "string" + }, + "to": { + "type": "string" + }, + "type": { + "enum": [ + "eql" + ], + "type": "string" + } + }, + "required": [ + "author", + "description", + "language", + "name", + "query", + "risk_score", + "rule_id", + "severity", + "type" + ], + "type": "object" +} \ No newline at end of file diff --git a/etc/api_schemas/8.0/8.0.machine_learning.json b/etc/api_schemas/8.0/8.0.machine_learning.json new file mode 100644 index 000000000..ccf730371 --- /dev/null +++ b/etc/api_schemas/8.0/8.0.machine_learning.json @@ -0,0 +1,347 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "additionalProperties": false, + "properties": { + "actions": { + "items": { + "type": [ + "string", + "number", + "object", + "array", + "boolean" + ] + }, + "type": "array" + }, + "anomaly_threshold": { + "format": "integer", + "type": "number" + }, + "author": { + "items": { + "type": "string" + }, + "type": "array" + }, + "building_block_type": { + "type": "string" + }, + "description": { + "type": "string" + }, + "enabled": { + "type": "boolean" + }, + "exceptions_list": { + "items": { + "type": [ + "string", + "number", + "object", + "array", + "boolean" + ] + }, + "type": "array" + }, + "false_positives": { + "items": { + "type": "string" + }, + "type": "array" + }, + "filters": { + "items": { + "additionalProperties": { + "type": [ + "string", + "number", + "object", + "array", + "boolean" + ] + }, + "type": "object" + }, + "type": "array" + }, + "from": { + "type": "string" + }, + "interval": { + "description": "Interval", + "pattern": "^\\d+[mshd]$", + "type": "string" + }, + "license": { + "type": "string" + }, + "machine_learning_job_id": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ] + }, + "max_signals": { + "description": "MaxSignals", + "format": "integer", + "minimum": 1, + "type": "number" + }, + "meta": { + "additionalProperties": { + "type": [ + "string", + "number", + "object", + "array", + "boolean" + ] + }, + "type": "object" + }, + "name": { + "type": "string" + }, + "note": { + "description": "MarkdownField", + "type": "string" + }, + "references": { + "items": { + "type": "string" + }, + "type": "array" + }, + "risk_score": { + "description": "MaxSignals", + "format": "integer", + "maximum": 100, + "minimum": 1, + "type": "number" + }, + "risk_score_mapping": { + "items": { + "additionalProperties": false, + "properties": { + "field": { + "type": "string" + }, + "operator": { + "enum": [ + "equals" + ], + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "field" + ], + "type": "object" + }, + "type": "array" + }, + "rule_id": { + "description": "UUIDString", + "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", + "type": "string" + }, + "rule_name_override": { + "type": "string" + }, + "severity": { + "enum": [ + "low", + "medium", + "high", + "critical" + ], + "enumNames": [], + "type": "string" + }, + "severity_mapping": { + "items": { + "additionalProperties": false, + "properties": { + "field": { + "type": "string" + }, + "operator": { + "enum": [ + "equals" + ], + "type": "string" + }, + "severity": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "field" + ], + "type": "object" + }, + "type": "array" + }, + "tags": { + "items": { + "type": "string" + }, + "type": "array" + }, + "threat": { + "items": { + "additionalProperties": false, + "properties": { + "framework": { + "enum": [ + "MITRE ATT&CK" + ], + "type": "string" + }, + "tactic": { + "additionalProperties": false, + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "reference": { + "description": "TacticURL", + "pattern": "^https://attack.mitre.org/tactics/TA[0-9]+/$", + "type": "string" + } + }, + "required": [ + "id", + "name", + "reference" + ], + "type": "object" + }, + "technique": { + "items": { + "additionalProperties": false, + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "reference": { + "description": "TechniqueURL", + "pattern": "^https://attack.mitre.org/techniques/T[0-9]+/$", + "type": "string" + }, + "subtechnique": { + "items": { + "additionalProperties": false, + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "reference": { + "description": "SubTechniqueURL", + "pattern": "^https://attack.mitre.org/techniques/T[0-9]+/[0-9]+/$", + "type": "string" + } + }, + "required": [ + "id", + "name", + "reference" + ], + "type": "object" + }, + "type": "array" + } + }, + "required": [ + "id", + "name", + "reference" + ], + "type": "object" + }, + "type": "array" + } + }, + "required": [ + "framework", + "tactic" + ], + "type": "object" + }, + "type": "array" + }, + "throttle": { + "type": "string" + }, + "timeline_id": { + "description": "TimelineTemplateId", + "enum": [ + "db366523-f1c6-4c1f-8731-6ce5ed9e5717", + "91832785-286d-4ebe-b884-1a208d111a70", + "76e52245-7519-4251-91ab-262fb1a1728c", + "495ad7a7-316e-4544-8a0f-9c098daee76e" + ], + "enumNames": [], + "type": "string" + }, + "timeline_title": { + "description": "TimelineTemplateTitle", + "enum": [ + "Generic Endpoint Timeline", + "Generic Network Timeline", + "Generic Process Timeline", + "Generic Threat Match Timeline" + ], + "enumNames": [], + "type": "string" + }, + "timestamp_override": { + "type": "string" + }, + "to": { + "type": "string" + }, + "type": { + "enum": [ + "machine_learning" + ], + "type": "string" + } + }, + "required": [ + "anomaly_threshold", + "author", + "description", + "machine_learning_job_id", + "name", + "risk_score", + "rule_id", + "severity", + "type" + ], + "type": "object" +} \ No newline at end of file diff --git a/etc/api_schemas/8.0/8.0.query.json b/etc/api_schemas/8.0/8.0.query.json new file mode 100644 index 000000000..78cbdae7a --- /dev/null +++ b/etc/api_schemas/8.0/8.0.query.json @@ -0,0 +1,347 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "additionalProperties": false, + "properties": { + "actions": { + "items": { + "type": [ + "string", + "number", + "object", + "array", + "boolean" + ] + }, + "type": "array" + }, + "author": { + "items": { + "type": "string" + }, + "type": "array" + }, + "building_block_type": { + "type": "string" + }, + "description": { + "type": "string" + }, + "enabled": { + "type": "boolean" + }, + "exceptions_list": { + "items": { + "type": [ + "string", + "number", + "object", + "array", + "boolean" + ] + }, + "type": "array" + }, + "false_positives": { + "items": { + "type": "string" + }, + "type": "array" + }, + "filters": { + "items": { + "additionalProperties": { + "type": [ + "string", + "number", + "object", + "array", + "boolean" + ] + }, + "type": "object" + }, + "type": "array" + }, + "from": { + "type": "string" + }, + "index": { + "items": { + "type": "string" + }, + "type": "array" + }, + "interval": { + "description": "Interval", + "pattern": "^\\d+[mshd]$", + "type": "string" + }, + "language": { + "enum": [ + "kuery", + "lucene" + ], + "enumNames": [], + "type": "string" + }, + "license": { + "type": "string" + }, + "max_signals": { + "description": "MaxSignals", + "format": "integer", + "minimum": 1, + "type": "number" + }, + "meta": { + "additionalProperties": { + "type": [ + "string", + "number", + "object", + "array", + "boolean" + ] + }, + "type": "object" + }, + "name": { + "type": "string" + }, + "note": { + "description": "MarkdownField", + "type": "string" + }, + "query": { + "type": "string" + }, + "references": { + "items": { + "type": "string" + }, + "type": "array" + }, + "risk_score": { + "description": "MaxSignals", + "format": "integer", + "maximum": 100, + "minimum": 1, + "type": "number" + }, + "risk_score_mapping": { + "items": { + "additionalProperties": false, + "properties": { + "field": { + "type": "string" + }, + "operator": { + "enum": [ + "equals" + ], + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "field" + ], + "type": "object" + }, + "type": "array" + }, + "rule_id": { + "description": "UUIDString", + "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", + "type": "string" + }, + "rule_name_override": { + "type": "string" + }, + "severity": { + "enum": [ + "low", + "medium", + "high", + "critical" + ], + "enumNames": [], + "type": "string" + }, + "severity_mapping": { + "items": { + "additionalProperties": false, + "properties": { + "field": { + "type": "string" + }, + "operator": { + "enum": [ + "equals" + ], + "type": "string" + }, + "severity": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "field" + ], + "type": "object" + }, + "type": "array" + }, + "tags": { + "items": { + "type": "string" + }, + "type": "array" + }, + "threat": { + "items": { + "additionalProperties": false, + "properties": { + "framework": { + "enum": [ + "MITRE ATT&CK" + ], + "type": "string" + }, + "tactic": { + "additionalProperties": false, + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "reference": { + "description": "TacticURL", + "pattern": "^https://attack.mitre.org/tactics/TA[0-9]+/$", + "type": "string" + } + }, + "required": [ + "id", + "name", + "reference" + ], + "type": "object" + }, + "technique": { + "items": { + "additionalProperties": false, + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "reference": { + "description": "TechniqueURL", + "pattern": "^https://attack.mitre.org/techniques/T[0-9]+/$", + "type": "string" + }, + "subtechnique": { + "items": { + "additionalProperties": false, + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "reference": { + "description": "SubTechniqueURL", + "pattern": "^https://attack.mitre.org/techniques/T[0-9]+/[0-9]+/$", + "type": "string" + } + }, + "required": [ + "id", + "name", + "reference" + ], + "type": "object" + }, + "type": "array" + } + }, + "required": [ + "id", + "name", + "reference" + ], + "type": "object" + }, + "type": "array" + } + }, + "required": [ + "framework", + "tactic" + ], + "type": "object" + }, + "type": "array" + }, + "throttle": { + "type": "string" + }, + "timeline_id": { + "description": "TimelineTemplateId", + "enum": [ + "db366523-f1c6-4c1f-8731-6ce5ed9e5717", + "91832785-286d-4ebe-b884-1a208d111a70", + "76e52245-7519-4251-91ab-262fb1a1728c", + "495ad7a7-316e-4544-8a0f-9c098daee76e" + ], + "enumNames": [], + "type": "string" + }, + "timeline_title": { + "description": "TimelineTemplateTitle", + "enum": [ + "Generic Endpoint Timeline", + "Generic Network Timeline", + "Generic Process Timeline", + "Generic Threat Match Timeline" + ], + "enumNames": [], + "type": "string" + }, + "timestamp_override": { + "type": "string" + }, + "to": { + "type": "string" + }, + "type": { + "enum": [ + "query" + ], + "type": "string" + } + }, + "required": [ + "author", + "description", + "language", + "name", + "query", + "risk_score", + "rule_id", + "severity", + "type" + ], + "type": "object" +} \ No newline at end of file diff --git a/etc/api_schemas/8.0/8.0.threat_match.json b/etc/api_schemas/8.0/8.0.threat_match.json new file mode 100644 index 000000000..c46b4d6dd --- /dev/null +++ b/etc/api_schemas/8.0/8.0.threat_match.json @@ -0,0 +1,438 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "additionalProperties": false, + "properties": { + "actions": { + "items": { + "type": [ + "string", + "number", + "object", + "array", + "boolean" + ] + }, + "type": "array" + }, + "author": { + "items": { + "type": "string" + }, + "type": "array" + }, + "building_block_type": { + "type": "string" + }, + "concurrent_searches": { + "description": "PositiveInteger", + "format": "integer", + "minimum": 1, + "type": "number" + }, + "description": { + "type": "string" + }, + "enabled": { + "type": "boolean" + }, + "exceptions_list": { + "items": { + "type": [ + "string", + "number", + "object", + "array", + "boolean" + ] + }, + "type": "array" + }, + "false_positives": { + "items": { + "type": "string" + }, + "type": "array" + }, + "filters": { + "items": { + "additionalProperties": { + "type": [ + "string", + "number", + "object", + "array", + "boolean" + ] + }, + "type": "object" + }, + "type": "array" + }, + "from": { + "type": "string" + }, + "index": { + "items": { + "type": "string" + }, + "type": "array" + }, + "interval": { + "description": "Interval", + "pattern": "^\\d+[mshd]$", + "type": "string" + }, + "items_per_search": { + "description": "PositiveInteger", + "format": "integer", + "minimum": 1, + "type": "number" + }, + "language": { + "enum": [ + "kuery", + "lucene" + ], + "enumNames": [], + "type": "string" + }, + "license": { + "type": "string" + }, + "max_signals": { + "description": "MaxSignals", + "format": "integer", + "minimum": 1, + "type": "number" + }, + "meta": { + "additionalProperties": { + "type": [ + "string", + "number", + "object", + "array", + "boolean" + ] + }, + "type": "object" + }, + "name": { + "type": "string" + }, + "note": { + "description": "MarkdownField", + "type": "string" + }, + "query": { + "type": "string" + }, + "references": { + "items": { + "type": "string" + }, + "type": "array" + }, + "risk_score": { + "description": "MaxSignals", + "format": "integer", + "maximum": 100, + "minimum": 1, + "type": "number" + }, + "risk_score_mapping": { + "items": { + "additionalProperties": false, + "properties": { + "field": { + "type": "string" + }, + "operator": { + "enum": [ + "equals" + ], + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "field" + ], + "type": "object" + }, + "type": "array" + }, + "rule_id": { + "description": "UUIDString", + "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", + "type": "string" + }, + "rule_name_override": { + "type": "string" + }, + "severity": { + "enum": [ + "low", + "medium", + "high", + "critical" + ], + "enumNames": [], + "type": "string" + }, + "severity_mapping": { + "items": { + "additionalProperties": false, + "properties": { + "field": { + "type": "string" + }, + "operator": { + "enum": [ + "equals" + ], + "type": "string" + }, + "severity": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "field" + ], + "type": "object" + }, + "type": "array" + }, + "tags": { + "items": { + "type": "string" + }, + "type": "array" + }, + "threat": { + "items": { + "additionalProperties": false, + "properties": { + "framework": { + "enum": [ + "MITRE ATT&CK" + ], + "type": "string" + }, + "tactic": { + "additionalProperties": false, + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "reference": { + "description": "TacticURL", + "pattern": "^https://attack.mitre.org/tactics/TA[0-9]+/$", + "type": "string" + } + }, + "required": [ + "id", + "name", + "reference" + ], + "type": "object" + }, + "technique": { + "items": { + "additionalProperties": false, + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "reference": { + "description": "TechniqueURL", + "pattern": "^https://attack.mitre.org/techniques/T[0-9]+/$", + "type": "string" + }, + "subtechnique": { + "items": { + "additionalProperties": false, + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "reference": { + "description": "SubTechniqueURL", + "pattern": "^https://attack.mitre.org/techniques/T[0-9]+/[0-9]+/$", + "type": "string" + } + }, + "required": [ + "id", + "name", + "reference" + ], + "type": "object" + }, + "type": "array" + } + }, + "required": [ + "id", + "name", + "reference" + ], + "type": "object" + }, + "type": "array" + } + }, + "required": [ + "framework", + "tactic" + ], + "type": "object" + }, + "type": "array" + }, + "threat_filters": { + "items": { + "additionalProperties": { + "type": [ + "string", + "number", + "object", + "array", + "boolean" + ] + }, + "type": "object" + }, + "type": "array" + }, + "threat_index": { + "items": { + "type": "string" + }, + "type": "array" + }, + "threat_indicator_path": { + "type": "string" + }, + "threat_language": { + "enum": [ + "kuery", + "lucene" + ], + "enumNames": [], + "type": "string" + }, + "threat_mapping": { + "items": { + "additionalProperties": false, + "properties": { + "entries": { + "items": { + "additionalProperties": false, + "properties": { + "field": { + "description": "NonEmptyStr", + "minLength": 1, + "type": "string" + }, + "type": { + "enum": [ + "mapping" + ], + "type": "string" + }, + "value": { + "description": "NonEmptyStr", + "minLength": 1, + "type": "string" + } + }, + "required": [ + "field", + "type", + "value" + ], + "type": "object" + }, + "type": "array" + } + }, + "required": [ + "entries" + ], + "type": "object" + }, + "type": "array" + }, + "threat_query": { + "type": "string" + }, + "throttle": { + "type": "string" + }, + "timeline_id": { + "description": "TimelineTemplateId", + "enum": [ + "db366523-f1c6-4c1f-8731-6ce5ed9e5717", + "91832785-286d-4ebe-b884-1a208d111a70", + "76e52245-7519-4251-91ab-262fb1a1728c", + "495ad7a7-316e-4544-8a0f-9c098daee76e" + ], + "enumNames": [], + "type": "string" + }, + "timeline_title": { + "description": "TimelineTemplateTitle", + "enum": [ + "Generic Endpoint Timeline", + "Generic Network Timeline", + "Generic Process Timeline", + "Generic Threat Match Timeline" + ], + "enumNames": [], + "type": "string" + }, + "timestamp_override": { + "type": "string" + }, + "to": { + "type": "string" + }, + "type": { + "enum": [ + "threat_match" + ], + "type": "string" + } + }, + "required": [ + "author", + "description", + "language", + "name", + "query", + "risk_score", + "rule_id", + "severity", + "threat_index", + "threat_mapping", + "type" + ], + "type": "object" +} \ No newline at end of file diff --git a/etc/api_schemas/8.0/8.0.threshold.json b/etc/api_schemas/8.0/8.0.threshold.json new file mode 100644 index 000000000..e755b4616 --- /dev/null +++ b/etc/api_schemas/8.0/8.0.threshold.json @@ -0,0 +1,396 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "additionalProperties": false, + "properties": { + "actions": { + "items": { + "type": [ + "string", + "number", + "object", + "array", + "boolean" + ] + }, + "type": "array" + }, + "author": { + "items": { + "type": "string" + }, + "type": "array" + }, + "building_block_type": { + "type": "string" + }, + "description": { + "type": "string" + }, + "enabled": { + "type": "boolean" + }, + "exceptions_list": { + "items": { + "type": [ + "string", + "number", + "object", + "array", + "boolean" + ] + }, + "type": "array" + }, + "false_positives": { + "items": { + "type": "string" + }, + "type": "array" + }, + "filters": { + "items": { + "additionalProperties": { + "type": [ + "string", + "number", + "object", + "array", + "boolean" + ] + }, + "type": "object" + }, + "type": "array" + }, + "from": { + "type": "string" + }, + "index": { + "items": { + "type": "string" + }, + "type": "array" + }, + "interval": { + "description": "Interval", + "pattern": "^\\d+[mshd]$", + "type": "string" + }, + "language": { + "enum": [ + "kuery", + "lucene" + ], + "enumNames": [], + "type": "string" + }, + "license": { + "type": "string" + }, + "max_signals": { + "description": "MaxSignals", + "format": "integer", + "minimum": 1, + "type": "number" + }, + "meta": { + "additionalProperties": { + "type": [ + "string", + "number", + "object", + "array", + "boolean" + ] + }, + "type": "object" + }, + "name": { + "type": "string" + }, + "note": { + "description": "MarkdownField", + "type": "string" + }, + "query": { + "type": "string" + }, + "references": { + "items": { + "type": "string" + }, + "type": "array" + }, + "risk_score": { + "description": "MaxSignals", + "format": "integer", + "maximum": 100, + "minimum": 1, + "type": "number" + }, + "risk_score_mapping": { + "items": { + "additionalProperties": false, + "properties": { + "field": { + "type": "string" + }, + "operator": { + "enum": [ + "equals" + ], + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "field" + ], + "type": "object" + }, + "type": "array" + }, + "rule_id": { + "description": "UUIDString", + "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", + "type": "string" + }, + "rule_name_override": { + "type": "string" + }, + "severity": { + "enum": [ + "low", + "medium", + "high", + "critical" + ], + "enumNames": [], + "type": "string" + }, + "severity_mapping": { + "items": { + "additionalProperties": false, + "properties": { + "field": { + "type": "string" + }, + "operator": { + "enum": [ + "equals" + ], + "type": "string" + }, + "severity": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "field" + ], + "type": "object" + }, + "type": "array" + }, + "tags": { + "items": { + "type": "string" + }, + "type": "array" + }, + "threat": { + "items": { + "additionalProperties": false, + "properties": { + "framework": { + "enum": [ + "MITRE ATT&CK" + ], + "type": "string" + }, + "tactic": { + "additionalProperties": false, + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "reference": { + "description": "TacticURL", + "pattern": "^https://attack.mitre.org/tactics/TA[0-9]+/$", + "type": "string" + } + }, + "required": [ + "id", + "name", + "reference" + ], + "type": "object" + }, + "technique": { + "items": { + "additionalProperties": false, + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "reference": { + "description": "TechniqueURL", + "pattern": "^https://attack.mitre.org/techniques/T[0-9]+/$", + "type": "string" + }, + "subtechnique": { + "items": { + "additionalProperties": false, + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "reference": { + "description": "SubTechniqueURL", + "pattern": "^https://attack.mitre.org/techniques/T[0-9]+/[0-9]+/$", + "type": "string" + } + }, + "required": [ + "id", + "name", + "reference" + ], + "type": "object" + }, + "type": "array" + } + }, + "required": [ + "id", + "name", + "reference" + ], + "type": "object" + }, + "type": "array" + } + }, + "required": [ + "framework", + "tactic" + ], + "type": "object" + }, + "type": "array" + }, + "threshold": { + "additionalProperties": false, + "properties": { + "cardinality": { + "items": { + "additionalProperties": false, + "properties": { + "field": { + "type": "string" + }, + "value": { + "description": "ThresholdValue", + "format": "integer", + "minimum": 1, + "type": "number" + } + }, + "required": [ + "field", + "value" + ], + "type": "object" + }, + "type": "array" + }, + "field": { + "description": "CardinalityFields", + "items": { + "description": "NonEmptyStr", + "minLength": 1, + "type": "string" + }, + "maxItems": 3, + "type": "array" + }, + "value": { + "description": "ThresholdValue", + "format": "integer", + "minimum": 1, + "type": "number" + } + }, + "required": [ + "field", + "value" + ], + "type": "object" + }, + "throttle": { + "type": "string" + }, + "timeline_id": { + "description": "TimelineTemplateId", + "enum": [ + "db366523-f1c6-4c1f-8731-6ce5ed9e5717", + "91832785-286d-4ebe-b884-1a208d111a70", + "76e52245-7519-4251-91ab-262fb1a1728c", + "495ad7a7-316e-4544-8a0f-9c098daee76e" + ], + "enumNames": [], + "type": "string" + }, + "timeline_title": { + "description": "TimelineTemplateTitle", + "enum": [ + "Generic Endpoint Timeline", + "Generic Network Timeline", + "Generic Process Timeline", + "Generic Threat Match Timeline" + ], + "enumNames": [], + "type": "string" + }, + "timestamp_override": { + "type": "string" + }, + "to": { + "type": "string" + }, + "type": { + "enum": [ + "threshold" + ], + "type": "string" + } + }, + "required": [ + "author", + "description", + "language", + "name", + "query", + "risk_score", + "rule_id", + "severity", + "threshold", + "type" + ], + "type": "object" +} \ No newline at end of file diff --git a/etc/beats_schemas/v7.16.2.json.gz b/etc/beats_schemas/v7.16.2.json.gz new file mode 100644 index 000000000..726fec0f4 Binary files /dev/null and b/etc/beats_schemas/v7.16.2.json.gz differ diff --git a/etc/beats_schemas/v8.0.0-rc1.json.gz b/etc/beats_schemas/v8.0.0-rc1.json.gz new file mode 100644 index 000000000..26d7aa4ef Binary files /dev/null and b/etc/beats_schemas/v8.0.0-rc1.json.gz differ diff --git a/etc/ecs_schemas/master_8.0.0.dev/ecs_flat.json.gz b/etc/ecs_schemas/master_8.0.0.dev/ecs_flat.json.gz deleted file mode 100644 index 7a7d4a856..000000000 Binary files a/etc/ecs_schemas/master_8.0.0.dev/ecs_flat.json.gz and /dev/null differ diff --git a/etc/ecs_schemas/master_8.2.0.dev/ecs_flat.json.gz b/etc/ecs_schemas/master_8.2.0.dev/ecs_flat.json.gz new file mode 100644 index 000000000..326eb2e21 Binary files /dev/null and b/etc/ecs_schemas/master_8.2.0.dev/ecs_flat.json.gz differ diff --git a/etc/packages.yml b/etc/packages.yml index e84ef0663..fd8615f35 100644 --- a/etc/packages.yml +++ b/etc/packages.yml @@ -1,6 +1,6 @@ --- package: - name: "8.0" + name: "8.1" release: true # exclude rules which have any of the following index <-> field pairs # exclude_fields: @@ -26,7 +26,7 @@ package: registry_data: categories: ["security"] conditions: - kibana.version: "^8.0.0" + kibana.version: "^8.0.1" description: Prebuilt detection rules for Elastic Security format_version: 1.0.0 icons: @@ -40,4 +40,4 @@ package: release: ga title: Prebuilt Security Detection Rules type: integration - version: 1.0.0-dev.0 + version: 1.0.1-dev.0 diff --git a/etc/stack-schema-map.yaml b/etc/stack-schema-map.yaml index 860c1fae6..27a04ac65 100644 --- a/etc/stack-schema-map.yaml +++ b/etc/stack-schema-map.yaml @@ -18,9 +18,15 @@ ecs: "1.11.0" "7.16.0": - beats: "7.15.1" # TODO: update this once beats releases + beats: "7.16.2" # TODO: update this once beats releases ecs: "1.12.1" +# 7.17 was intentionally skipped because it was added late and was bug fix only + "8.0.0": + beats: "8.0.0-rc1" # TODO: update this once beats releases + ecs: "1.12.1" + +"8.1.0": beats: "master" # TODO: update this once beats releases ecs: "1.12.1"