From c567d3731af91ddd7616c1ba008d427da6868da6 Mon Sep 17 00:00:00 2001 From: Justin Ibarra <16747370+brokensound77@users.noreply.github.com> Date: Fri, 26 Apr 2024 11:12:50 -0600 Subject: [PATCH] Refresh Kibana module with API updates (#3466) * Refresh Kibana module with API updates * add import/export commands * rename repo commands * add RawRuleCollection and DictRule objects * save exported rules to files; rule.from_rule_resource * strip unknown fields in schema * add remote cli test * update docs * bump kibana lib version --------- Co-authored-by: brokensound77 --- CLI.md | 266 +++++++++++++++++++++++ detection_rules/cli_utils.py | 6 +- detection_rules/etc/test_remote_cli.bash | 7 + detection_rules/kbwrap.py | 103 ++++++++- detection_rules/main.py | 18 +- detection_rules/mixins.py | 47 +++- detection_rules/rule.py | 48 ++++ detection_rules/rule_loader.py | 175 ++++++++++++++- detection_rules/utils.py | 10 + docs/developing.md | 24 ++ lib/kibana/kibana/__init__.py | 2 +- lib/kibana/kibana/connector.py | 28 ++- lib/kibana/kibana/definitions.py | 88 ++++++++ lib/kibana/kibana/resources.py | 134 +++++++++++- lib/kibana/pyproject.toml | 2 +- 15 files changed, 914 insertions(+), 44 deletions(-) create mode 100644 lib/kibana/kibana/definitions.py diff --git a/CLI.md b/CLI.md index 858abae03..f7da7b7e0 100644 --- a/CLI.md +++ b/CLI.md @@ -239,6 +239,8 @@ Toml formatted rule files can be uploaded as custom rules using the `kibana uplo file, specify multiple files at a time as individual args. This command is meant to support uploading and testing of rules and is not intended for production use in its current state. +This command is built on soon to be deprecated APIs and so should be phased off. For a better option, see below... + ```console python -m detection_rules kibana upload-rule -h @@ -290,6 +292,270 @@ _*To load a custom rule, the proper index must be setup first. The simplest way the `Load prebuilt detection rules and timeline templates` button on the `detections` page in the Kibana security app._ +### Using `import-rules` + +This is a better option than `upload-rule` as it is built on refreshed APIs + +``` +python -m detection_rules kibana import-rules -h + +█▀▀▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄ ▄ █▀▀▄ ▄ ▄ ▄ ▄▄▄ ▄▄▄ +█ █ █▄▄ █ █▄▄ █ █ █ █ █ █▀▄ █ █▄▄▀ █ █ █ █▄▄ █▄▄ +█▄▄▀ █▄▄ █ █▄▄ █▄▄ █ ▄█▄ █▄█ █ ▀▄█ █ ▀▄ █▄▄█ █▄▄ █▄▄ ▄▄█ + +Kibana client: +Options: + --ignore-ssl-errors TEXT + --space TEXT Kibana space + --provider-name TEXT Elastic Cloud providers: cloud-basic and cloud- + saml (for SSO) + --provider-type TEXT Elastic Cloud providers: basic and saml (for + SSO) + -ku, --kibana-user TEXT + --kibana-url TEXT + -kp, --kibana-password TEXT + -kc, --kibana-cookie TEXT Cookie from an authed session + --cloud-id TEXT ID of the cloud instance. + +Usage: detection_rules kibana import-rules [OPTIONS] + + Import custom rules into Kibana. + +Options: + -f, --rule-file FILE + -d, --directory DIRECTORY Recursively load rules from a directory + -id, --rule-id TEXT + -o, --overwrite Overwrite existing rules + -e, --overwrite-exceptions Overwrite exceptions in existing rules + -a, --overwrite-action-connectors + Overwrite action connectors in existing + rules + -h, --help Show this message and exit. +``` + +Example usage of a successful upload: + +``` +python -m detection_rules kibana import-rules -f test-export-rules/credential_access_NEW_RULE.toml + +█▀▀▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄ ▄ █▀▀▄ ▄ ▄ ▄ ▄▄▄ ▄▄▄ +█ █ █▄▄ █ █▄▄ █ █ █ █ █ █▀▄ █ █▄▄▀ █ █ █ █▄▄ █▄▄ +█▄▄▀ █▄▄ █ █▄▄ █▄▄ █ ▄█▄ █▄█ █ ▀▄█ █ ▀▄ █▄▄█ █▄▄ █▄▄ ▄▄█ + +1 rule(s) successfully imported + - 50887ba8-aaaa-bbbb-a038-f661ea17fbcd +``` + +
+Detailed import-rules output + +Existing rule fails as expected: +``` +python -m detection_rules kibana import-rules -f test-export-rules/credential_access_EXISTING_RULE.toml + +█▀▀▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄ ▄ █▀▀▄ ▄ ▄ ▄ ▄▄▄ ▄▄▄ +█ █ █▄▄ █ █▄▄ █ █ █ █ █ █▀▄ █ █▄▄▀ █ █ █ █▄▄ █▄▄ +█▄▄▀ █▄▄ █ █▄▄ █▄▄ █ ▄█▄ █▄█ █ ▀▄█ █ ▀▄ █▄▄█ █▄▄ █▄▄ ▄▄█ + +1 rule(s) failed to import! + - 50887ba8-7ff7-11ee-a038-f661ea17fbcd: (409) rule_id: "50887ba8-7ff7-11ee-a038-f661ea17fbcd" already exists +``` + +`-o` overwrite forces the import successfully +``` +python -m detection_rules kibana import-rules -f test-export-rules/credential_access_EXISTING_RULE.toml -o + +█▀▀▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄ ▄ █▀▀▄ ▄ ▄ ▄ ▄▄▄ ▄▄▄ +█ █ █▄▄ █ █▄▄ █ █ █ █ █ █▀▄ █ █▄▄▀ █ █ █ █▄▄ █▄▄ +█▄▄▀ █▄▄ █ █▄▄ █▄▄ █ ▄█▄ █▄█ █ ▀▄█ █ ▀▄ █▄▄█ █▄▄ █▄▄ ▄▄█ + +1 rule(s) successfully imported + - 50887ba8-7ff7-11ee-a038-f661ea17fbcd +``` + +The rule loader detects a collision in name and fails as intended: +``` +python -m detection_rules kibana import-rules -d test-export-rules + +█▀▀▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄ ▄ █▀▀▄ ▄ ▄ ▄ ▄▄▄ ▄▄▄ +█ █ █▄▄ █ █▄▄ █ █ █ █ █ █▀▄ █ █▄▄▀ █ █ █ █▄▄ █▄▄ +█▄▄▀ █▄▄ █ █▄▄ █▄▄ █ ▄█▄ █▄█ █ ▀▄█ █ ▀▄ █▄▄█ █▄▄ █▄▄ ▄▄█ + +Error loading rule in test-export-rules/credential_access_NEW_RULE.toml +Traceback (most recent call last): + ...snipped stacktrace... +AssertionError: Rule Name Multiple Okta User Auth Events with Same Device Token Hash Behind a Proxy for 50887ba8-aaaa-bbbb-a038-f661ea17fbcd collides with rule ID 50887ba8-7ff7-11ee-a038-f661ea17fbcd +``` + +Expected failure on rule_id collision: +``` +python -m detection_rules kibana import-rules -d test-export-rules + +█▀▀▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄ ▄ █▀▀▄ ▄ ▄ ▄ ▄▄▄ ▄▄▄ +█ █ █▄▄ █ █▄▄ █ █ █ █ █ █▀▄ █ █▄▄▀ █ █ █ █▄▄ █▄▄ +█▄▄▀ █▄▄ █ █▄▄ █▄▄ █ ▄█▄ █▄█ █ ▀▄█ █ ▀▄ █▄▄█ █▄▄ █▄▄ ▄▄█ + +Error loading rule in test-export-rules/credential_access_multiple_okta_user_auth_events_with_same_device_token_hash_behind_a_proxy.toml +Traceback (most recent call last): + ...snipped stacktrace... +AssertionError: Rule ID 50887ba8-7ff7-11ee-a038-f661ea17fbcd for Multiple Okta User Auth Events with Same Device Token Hash Behind a Proxy collides with rule Multiple Okta User Auth Events with Same Device Token Hash Behind a Proxy +``` + +Import a full directory - all fail as expected: +``` +python -m detection_rules kibana import-rules -d test-export-rules + +█▀▀▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄ ▄ █▀▀▄ ▄ ▄ ▄ ▄▄▄ ▄▄▄ +█ █ █▄▄ █ █▄▄ █ █ █ █ █ █▀▄ █ █▄▄▀ █ █ █ █▄▄ █▄▄ +█▄▄▀ █▄▄ █ █▄▄ █▄▄ █ ▄█▄ █▄█ █ ▀▄█ █ ▀▄ █▄▄█ █▄▄ █▄▄ ▄▄█ + +23 rule(s) failed to import! + - ee663abc-fb77-49d2-a7c5-204b9cf888ca: (409) rule_id: "ee663abc-fb77-49d2-a7c5-204b9cf888ca" already exists + - 50887ba8-aaaa-bbbb-a038-f661ea17fbcd: (409) rule_id: "50887ba8-aaaa-bbbb-a038-f661ea17fbcd" already exists + - 50887ba8-7ff7-11ee-a038-f661ea17fbcd: (409) rule_id: "50887ba8-7ff7-11ee-a038-f661ea17fbcd" already exists + - 8a0fbd26-867f-11ee-947c-f661ea17fbcd: (409) rule_id: "8a0fbd26-867f-11ee-947c-f661ea17fbcd" already exists + - aaaaaaaa-f861-414c-8602-150d5505b777: (409) rule_id: "aaaaaaaa-f861-414c-8602-150d5505b777" already exists + - 2f8a1226-5720-437d-9c20-e0029deb6194: (409) rule_id: "2f8a1226-5720-437d-9c20-e0029deb6194" already exists + - cd66a5af-e34b-4bb0-8931-57d0a043f2ef: (409) rule_id: "cd66a5af-e34b-4bb0-8931-57d0a043f2ef" already exists + - 2d8043ed-5bda-4caf-801c-c1feb7410504: (409) rule_id: "2d8043ed-5bda-4caf-801c-c1feb7410504" already exists + - d76b02ef-fc95-4001-9297-01cb7412232f: (409) rule_id: "d76b02ef-fc95-4001-9297-01cb7412232f" already exists + - cc382a2e-7e52-11ee-9aac-f661ea17fbcd: (409) rule_id: "cc382a2e-7e52-11ee-9aac-f661ea17fbcd" already exists + - 260486ee-7d98-11ee-9599-f661ea17fbcd: (409) rule_id: "260486ee-7d98-11ee-9599-f661ea17fbcd" already exists + - ee39a9f7-5a79-4b0a-9815-d36b3cf28d3e: (409) rule_id: "ee39a9f7-5a79-4b0a-9815-d36b3cf28d3e" already exists + - 1ceb05c4-7d25-11ee-9562-f661ea17fbcd: (409) rule_id: "1ceb05c4-7d25-11ee-9562-f661ea17fbcd" already exists + - 2e56e1bc-867a-11ee-b13e-f661ea17fbcd: (409) rule_id: "2e56e1bc-867a-11ee-b13e-f661ea17fbcd" already exists + - 621e92b6-7e54-11ee-bdc0-f661ea17fbcd: (409) rule_id: "621e92b6-7e54-11ee-bdc0-f661ea17fbcd" already exists + - a198fbbd-9413-45ec-a269-47ae4ccf59ce: (409) rule_id: "a198fbbd-9413-45ec-a269-47ae4ccf59ce" already exists + - 29b53942-7cd4-11ee-b70e-f661ea17fbcd: (409) rule_id: "29b53942-7cd4-11ee-b70e-f661ea17fbcd" already exists + - aaec44bc-d691-4874-99b2-48ab7392dfd5: (409) rule_id: "aaec44bc-d691-4874-99b2-48ab7392dfd5" already exists + - 40e1f208-0f70-47d4-98ea-378ccf504ad3: (409) rule_id: "40e1f208-0f70-47d4-98ea-378ccf504ad3" already exists + - 5e9bc07c-7e7a-415b-a6c0-1cae4a0d256e: (409) rule_id: "5e9bc07c-7e7a-415b-a6c0-1cae4a0d256e" already exists + - 17d99572-793d-41ae-8b55-cee30db13fa2: (409) rule_id: "17d99572-793d-41ae-8b55-cee30db13fa2" already exists + - 38accba8-894a-4f32-98d5-7cb01c82f5d6: (409) rule_id: "38accba8-894a-4f32-98d5-7cb01c82f5d6" already exists + - e1b7d2a6-d23a-4747-b621-d249d83162ea: (409) rule_id: "e1b7d2a6-d23a-4747-b621-d249d83162ea" already exists +``` + +Import a full directory, with `-o` forcing the updates successfully +``` +python -m detection_rules kibana import-rules -d test-export-rules -o + +█▀▀▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄ ▄ █▀▀▄ ▄ ▄ ▄ ▄▄▄ ▄▄▄ +█ █ █▄▄ █ █▄▄ █ █ █ █ █ █▀▄ █ █▄▄▀ █ █ █ █▄▄ █▄▄ +█▄▄▀ █▄▄ █ █▄▄ █▄▄ █ ▄█▄ █▄█ █ ▀▄█ █ ▀▄ █▄▄█ █▄▄ █▄▄ ▄▄█ + +23 rule(s) successfully imported + - ee663abc-fb77-49d2-a7c5-204b9cf888ca + - 50887ba8-aaaa-bbbb-a038-f661ea17fbcd + - 50887ba8-7ff7-11ee-a038-f661ea17fbcd + - 8a0fbd26-867f-11ee-947c-f661ea17fbcd + - aaaaaaaa-f861-414c-8602-150d5505b777 + - 2f8a1226-5720-437d-9c20-e0029deb6194 + - cd66a5af-e34b-4bb0-8931-57d0a043f2ef + - 2d8043ed-5bda-4caf-801c-c1feb7410504 + - d76b02ef-fc95-4001-9297-01cb7412232f + - cc382a2e-7e52-11ee-9aac-f661ea17fbcd + - 260486ee-7d98-11ee-9599-f661ea17fbcd + - ee39a9f7-5a79-4b0a-9815-d36b3cf28d3e + - 1ceb05c4-7d25-11ee-9562-f661ea17fbcd + - 2e56e1bc-867a-11ee-b13e-f661ea17fbcd + - 621e92b6-7e54-11ee-bdc0-f661ea17fbcd + - a198fbbd-9413-45ec-a269-47ae4ccf59ce + - 29b53942-7cd4-11ee-b70e-f661ea17fbcd + - aaec44bc-d691-4874-99b2-48ab7392dfd5 + - 40e1f208-0f70-47d4-98ea-378ccf504ad3 + - 5e9bc07c-7e7a-415b-a6c0-1cae4a0d256e + - 17d99572-793d-41ae-8b55-cee30db13fa2 + - 38accba8-894a-4f32-98d5-7cb01c82f5d6 + - e1b7d2a6-d23a-4747-b621-d249d83162ea + +``` + +
+ +### Exporting rules + +Example of a rule exporting, with errors skipped + +``` +python -m detection_rules kibana export-rules -d test-export-rules --skip-errors + +█▀▀▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄ ▄ █▀▀▄ ▄ ▄ ▄ ▄▄▄ ▄▄▄ +█ █ █▄▄ █ █▄▄ █ █ █ █ █ █▀▄ █ █▄▄▀ █ █ █ █▄▄ █▄▄ +█▄▄▀ █▄▄ █ █▄▄ █▄▄ █ ▄█▄ █▄█ █ ▀▄█ █ ▀▄ █▄▄█ █▄▄ █▄▄ ▄▄█ + +- skipping Stolen Credentials Used to Login to Okta Account After MFA Reset - ValidationError +- skipping First Occurrence of Okta User Session Started via Proxy - ValidationError +- skipping ESQL test: cmd child of Explorer - ValidationError +- skipping Potential Persistence Through Run Control Detected - ValidationError +- skipping First Time Seen AWS Secret Value Accessed in Secrets Manager - ValidationError +- skipping Potential Shadow File Read via Command Line Utilities - ValidationError +- skipping Abnormal Process ID or Lock File Created - ValidationError +- skipping New service installed in last 24 hours - ValidationError +- skipping Scheduled Task or Driver added - KqlParseError +- skipping Scheduled Task or Driver removed - KqlParseError +- skipping name - ValidationError +33 rules exported +22 rules converted +22 saved to test-export-rules +11 errors saved to test-export-rules/_errors.txt +``` + +Directory of the output: + +``` +ls test-export-rules + +_errors.txt +collection_exchange_mailbox_export_via_powershell.toml.toml +credential_access_multiple_okta_user_auth_events_with_same_device_token_hash_behind_a_proxy.toml.toml +credential_access_potential_okta_mfa_bombing_via_push_notifications.toml.toml +defense_evasion_agent_spoofing_multiple_hosts_using_same_agent.toml.toml +defense_evasion_attempt_to_disable_syslog_service.toml.toml +defense_evasion_kernel_module_removal.toml.toml +discovery_enumeration_of_kernel_modules.toml.toml +execution_interactive_terminal_spawned_via_python.toml.toml +initial_access_multiple_okta_client_addresses_for_a_single_user_session.toml.toml +initial_access_new_okta_authentication_behavior_detected.toml.toml +initial_access_okta_fastpass_phishing_detection.toml.toml +initial_access_okta_sign_in_events_via_third_party_idp.toml.toml +initial_access_okta_user_sessions_started_from_different_geolocations.toml.toml +lateral_movement_multiple_okta_sessions_detected_for_a_single_user.toml.toml +my_first_alert.toml.toml +persistence_new_okta_identity_provider_idp_added_by_admin.toml.toml +test_data_view.toml.toml +test_noisy.toml.toml +test_suppress.toml.toml +web_application_suspicious_activity_post_request_declined.toml.toml +web_application_suspicious_activity_sqlmap_user_agent.toml.toml +web_application_suspicious_activity_unauthorized_method.toml.toml +``` + +Output of the `_errors.txt` file: + +``` +cat test-export-rules/_errors.txt +- Stolen Credentials Used to Login to Okta Account After MFA Reset - {'_schema': ['Setup header found in both note and setup fields.']} +- First Occurrence of Okta User Session Started via Proxy - {'rule': [ValidationError({'type': ['Must be equal to eql.'], 'language': ['Must be equal to eql.']}), ValidationError({'type': ['Must be equal to esql.'], 'language': ['Must be equal to esql.']}), ValidationError({'type': ['Must be equal to threshold.'], 'threshold': ['Missing data for required field.']}), ValidationError({'type': ['Must be equal to threat_match.'], 'threat_mapping': ['Missing data for required field.'], 'threat_index': ['Missing data for required field.']}), ValidationError({'type': ['Must be equal to machine_learning.'], 'anomaly_threshold': ['Missing data for required field.'], 'machine_learning_job_id': ['Missing data for required field.']}), ValidationError({'type': ['Must be equal to query.']}), ValidationError({'new_terms': ['Missing data for required field.']})]} +- ESQL test: cmd child of Explorer - {'rule': [ValidationError({'type': ['Must be equal to eql.'], 'threat': {0: {'tactic': {'reference': ['String does not match expected pattern.']}, 'technique': {0: {'reference': ['String does not match expected pattern.']}}}}, 'language': ['Must be equal to eql.']}), ValidationError({'threat': {0: {'tactic': {'reference': ['String does not match expected pattern.']}, 'technique': {0: {'reference': ['String does not match expected pattern.']}}}}}), ValidationError({'type': ['Must be equal to threshold.'], 'threat': {0: {'tactic': {'reference': ['String does not match expected pattern.']}, 'technique': {0: {'reference': ['String does not match expected pattern.']}}}}, 'threshold': ['Missing data for required field.']}), ValidationError({'type': ['Must be equal to threat_match.'], 'threat': {0: {'tactic': {'reference': ['String does not match expected pattern.']}, 'technique': {0: {'reference': ['String does not match expected pattern.']}}}}, 'threat_mapping': ['Missing data for required field.'], 'threat_index': ['Missing data for required field.']}), ValidationError({'type': ['Must be equal to machine_learning.'], 'threat': {0: {'tactic': {'reference': ['String does not match expected pattern.']}, 'technique': {0: {'reference': ['String does not match expected pattern.']}}}}, 'anomaly_threshold': ['Missing data for required field.'], 'machine_learning_job_id': ['Missing data for required field.']}), ValidationError({'type': ['Must be equal to query.'], 'threat': {0: {'tactic': {'reference': ['String does not match expected pattern.']}, 'technique': {0: {'reference': ['String does not match expected pattern.']}}}}}), ValidationError({'type': ['Must be equal to new_terms.'], 'threat': {0: {'tactic': {'reference': ['String does not match expected pattern.']}, 'technique': {0: {'reference': ['String does not match expected pattern.']}}}}, 'new_terms': ['Missing data for required field.']})]} +- Potential Persistence Through Run Control Detected - {'rule': [ValidationError({'type': ['Must be equal to eql.'], 'language': ['Must be equal to eql.']}), ValidationError({'type': ['Must be equal to esql.'], 'language': ['Must be equal to esql.']}), ValidationError({'type': ['Must be equal to threshold.'], 'threshold': ['Missing data for required field.']}), ValidationError({'type': ['Must be equal to threat_match.'], 'threat_mapping': ['Missing data for required field.'], 'threat_index': ['Missing data for required field.']}), ValidationError({'type': ['Must be equal to machine_learning.'], 'anomaly_threshold': ['Missing data for required field.'], 'machine_learning_job_id': ['Missing data for required field.']}), ValidationError({'type': ['Must be equal to query.']}), ValidationError({'new_terms': ['Missing data for required field.']})]} +- First Time Seen AWS Secret Value Accessed in Secrets Manager - {'rule': [ValidationError({'type': ['Must be equal to eql.'], 'language': ['Must be equal to eql.']}), ValidationError({'type': ['Must be equal to esql.'], 'language': ['Must be equal to esql.']}), ValidationError({'type': ['Must be equal to threshold.'], 'threshold': ['Missing data for required field.']}), ValidationError({'type': ['Must be equal to threat_match.'], 'threat_mapping': ['Missing data for required field.'], 'threat_index': ['Missing data for required field.']}), ValidationError({'type': ['Must be equal to machine_learning.'], 'anomaly_threshold': ['Missing data for required field.'], 'machine_learning_job_id': ['Missing data for required field.']}), ValidationError({'type': ['Must be equal to query.']}), ValidationError({'new_terms': ['Missing data for required field.']})]} +- Potential Shadow File Read via Command Line Utilities - {'rule': [ValidationError({'type': ['Must be equal to eql.'], 'language': ['Must be equal to eql.']}), ValidationError({'type': ['Must be equal to esql.'], 'language': ['Must be equal to esql.']}), ValidationError({'type': ['Must be equal to threshold.'], 'threshold': ['Missing data for required field.']}), ValidationError({'type': ['Must be equal to threat_match.'], 'threat_mapping': ['Missing data for required field.'], 'threat_index': ['Missing data for required field.']}), ValidationError({'type': ['Must be equal to machine_learning.'], 'anomaly_threshold': ['Missing data for required field.'], 'machine_learning_job_id': ['Missing data for required field.']}), ValidationError({'type': ['Must be equal to query.']}), ValidationError({'new_terms': ['Missing data for required field.']})]} +- Abnormal Process ID or Lock File Created - {'rule': [ValidationError({'type': ['Must be equal to eql.'], 'language': ['Must be equal to eql.']}), ValidationError({'type': ['Must be equal to esql.'], 'language': ['Must be equal to esql.']}), ValidationError({'type': ['Must be equal to threshold.'], 'threshold': ['Missing data for required field.']}), ValidationError({'type': ['Must be equal to threat_match.'], 'threat_mapping': ['Missing data for required field.'], 'threat_index': ['Missing data for required field.']}), ValidationError({'type': ['Must be equal to machine_learning.'], 'anomaly_threshold': ['Missing data for required field.'], 'machine_learning_job_id': ['Missing data for required field.']}), ValidationError({'type': ['Must be equal to query.']}), ValidationError({'new_terms': ['Missing data for required field.']})]} +- New service installed in last 24 hours - {'rule': [ValidationError({'type': ['Must be equal to eql.'], 'language': ['Must be equal to eql.']}), ValidationError({'type': ['Must be equal to esql.'], 'language': ['Must be equal to esql.']}), ValidationError({'type': ['Must be equal to threshold.'], 'threshold': ['Missing data for required field.']}), ValidationError({'type': ['Must be equal to threat_match.'], 'threat_mapping': ['Missing data for required field.'], 'threat_index': ['Missing data for required field.']}), ValidationError({'type': ['Must be equal to machine_learning.'], 'anomaly_threshold': ['Missing data for required field.'], 'machine_learning_job_id': ['Missing data for required field.']}), ValidationError({'type': ['Must be equal to query.']}), ValidationError({'new_terms': ['Missing data for required field.']})]} +- Scheduled Task or Driver added - Error at line:1,column:75 +Unknown field +data_stream.dataset:osquery_manager.result and osquery_meta.counter>0 and osquery_meta.type:diff and osquery.last_run_code:0 and osquery_meta.action:added + ^^^^^^^^^^^^^^^^^ +stack: 8.9.0, beats: 8.9.0, ecs: 8.9.0 +- Scheduled Task or Driver removed - Error at line:1,column:75 +Unknown field +data_stream.dataset:osquery_manager.result and osquery_meta.counter>0 and osquery_meta.type:diff and osquery.last_run_code:0 and osquery_meta.action:removed + ^^^^^^^^^^^^^^^^^ +stack: 8.9.0, beats: 8.9.0, ecs: 8.9.0 +- name - {'rule': [ValidationError({'type': ['Must be equal to eql.'], 'language': ['Must be equal to eql.']}), ValidationError({'type': ['Must be equal to esql.'], 'language': ['Must be equal to esql.']}), ValidationError({'type': ['Must be equal to threshold.'], 'threshold': ['Missing data for required field.']}), ValidationError({'type': ['Must be equal to threat_match.'], 'threat_mapping': ['Missing data for required field.'], 'threat_index': ['Missing data for required field.']}), ValidationError({'type': ['Must be equal to machine_learning.'], 'anomaly_threshold': ['Missing data for required field.'], 'machine_learning_job_id': ['Missing data for required field.']}), ValidationError({'type': ['Must be equal to query.']}), ValidationError({'new_terms': ['Missing data for required field.']})]}(venv312) ➜ detection-rules-fork git:(refresh-kibana-module-with-new-APIs) ✗ +``` + + ## Converting between JSON and TOML [Importing rules](#importing-rules-into-the-repo) will convert from any supported format to toml. Additionally, the diff --git a/detection_rules/cli_utils.py b/detection_rules/cli_utils.py index 710ae8b67..320e8dbaa 100644 --- a/detection_rules/cli_utils.py +++ b/detection_rules/cli_utils.py @@ -50,7 +50,7 @@ def single_collection(f): if rule_id: rules.load_directories((DEFAULT_RULES_DIR, DEFAULT_BBR_DIR), - toml_filter=dict_filter(rule__rule_id=rule_id)) + obj_filter=dict_filter(rule__rule_id=rule_id)) if len(rules) != 1: client_error(f"Could not find rule with ID {rule_id}") @@ -66,7 +66,7 @@ def multi_collection(f): @click.option('--rule-file', '-f', multiple=True, type=click.Path(dir_okay=False), required=False) @click.option('--directory', '-d', multiple=True, type=click.Path(file_okay=False), required=False, - help='Recursively export rules from a directory') + help='Recursively load rules from a directory') @click.option('--rule-id', '-id', multiple=True, required=False) @functools.wraps(f) def get_collection(*args, **kwargs): @@ -84,7 +84,7 @@ def multi_collection(f): if rule_id: rules.load_directories((DEFAULT_RULES_DIR, DEFAULT_BBR_DIR), - toml_filter=dict_filter(rule__rule_id=rule_id)) + obj_filter=dict_filter(rule__rule_id=rule_id)) found_ids = {rule.id for rule in rules} missing = set(rule_id).difference(found_ids) diff --git a/detection_rules/etc/test_remote_cli.bash b/detection_rules/etc/test_remote_cli.bash index 2d9eccc63..235873a71 100755 --- a/detection_rules/etc/test_remote_cli.bash +++ b/detection_rules/etc/test_remote_cli.bash @@ -12,4 +12,11 @@ echo "Performing a quick rule alerts search..." echo "Requires .detection-rules-cfg.json credentials file set." python -m detection_rules kibana search-alerts +echo "Performing a rule export..." +mkdir tmp-export 2>/dev/null +python -m detection_rules kibana export-rules -d tmp-export --skip-errors +ls tmp-export +echo "Removing generated files..." +rm -rf tmp-export + echo "Detection-rules CLI tests completed!" diff --git a/detection_rules/kbwrap.py b/detection_rules/kbwrap.py index 44d655ef0..48f3b775c 100644 --- a/detection_rules/kbwrap.py +++ b/detection_rules/kbwrap.py @@ -5,17 +5,20 @@ """Kibana cli commands.""" import sys +from pathlib import Path +from typing import Iterable, List, Optional import click - import kql from kibana import Signal, RuleResource + from .cli_utils import multi_collection from .main import root from .misc import add_params, client_error, kibana_options, get_kibana_client, nested_set -from .rule import downgrade_contents_from_rule -from .utils import format_command_options +from .rule import downgrade_contents_from_rule, TOMLRuleContents, TOMLRule +from .rule_loader import RuleCollection +from .utils import format_command_options, rulename_to_filename @root.group('kibana') @@ -38,7 +41,7 @@ def kibana_group(ctx: click.Context, **kibana_kwargs): @multi_collection @click.option('--replace-id', '-r', is_flag=True, help='Replace rule IDs with new IDs before export') @click.pass_context -def upload_rule(ctx, rules, replace_id): +def upload_rule(ctx, rules: RuleCollection, replace_id): """Upload a list of rule .toml files to Kibana.""" kibana = ctx.obj['kibana'] api_payloads = [] @@ -53,7 +56,7 @@ def upload_rule(ctx, rules, replace_id): api_payloads.append(rule) with kibana: - results = RuleResource.bulk_create(api_payloads) + results = RuleResource.bulk_create_legacy(api_payloads) success = [] errors = [] @@ -71,6 +74,96 @@ def upload_rule(ctx, rules, replace_id): return results +@kibana_group.command('import-rules') +@multi_collection +@click.option('--overwrite', '-o', is_flag=True, help='Overwrite existing rules') +@click.option('--overwrite-exceptions', '-e', is_flag=True, help='Overwrite exceptions in existing rules') +@click.option('--overwrite-action-connectors', '-a', is_flag=True, + help='Overwrite action connectors in existing rules') +@click.pass_context +def kibana_import_rules(ctx: click.Context, rules: RuleCollection, overwrite: Optional[bool] = False, + overwrite_exceptions: Optional[bool] = False, + overwrite_action_connectors: Optional[bool] = False) -> (dict, List[RuleResource]): + """Import custom rules into Kibana.""" + kibana = ctx.obj['kibana'] + rule_dicts = [r.contents.to_api_format() for r in rules] + with kibana: + response, successful_rule_ids, results = RuleResource.import_rules( + rule_dicts, + overwrite=overwrite, + overwrite_exceptions=overwrite_exceptions, + overwrite_action_connectors=overwrite_action_connectors + ) + + if successful_rule_ids: + click.echo(f'{len(successful_rule_ids)} rule(s) successfully imported') + rule_str = '\n - '.join(successful_rule_ids) + print(f' - {rule_str}') + if response['errors']: + click.echo(f'{len(response["errors"])} rule(s) failed to import!') + for error in response['errors']: + click.echo(f' - {error["rule_id"]}: ({error["error"]["status_code"]}) {error["error"]["message"]}') + + return response, results + + +@kibana_group.command('export-rules') +@click.option('--directory', '-d', required=True, type=Path, help='Directory to export rules to') +@click.option('--rule-id', '-r', multiple=True, help='Optional Rule IDs to restrict export to') +@click.option('--skip-errors', '-s', is_flag=True, help='Skip errors when exporting rules') +@click.pass_context +def kibana_export_rules(ctx: click.Context, directory: Path, + rule_id: Optional[Iterable[str]] = None, skip_errors: bool = False) -> List[TOMLRule]: + """Export custom rules from Kibana.""" + kibana = ctx.obj['kibana'] + with kibana: + results = RuleResource.export_rules(list(rule_id)) + + if results: + directory.mkdir(parents=True, exist_ok=True) + + errors = [] + exported = [] + for rule_resource in results: + try: + contents = TOMLRuleContents.from_rule_resource(rule_resource, maturity='production') + threat = contents.data.get('threat') + first_tactic = threat[0].tactic.name if threat else '' + rule_name = rulename_to_filename(contents.data.name, tactic_name=first_tactic) + rule = TOMLRule(contents=contents, path=directory / f'{rule_name}') + except Exception as e: + if skip_errors: + print(f'- skipping {rule_resource.get("name")} - {type(e).__name__}') + errors.append(f'- {rule_resource.get("name")} - {e}') + continue + raise + + exported.append(rule) + + saved = [] + for rule in exported: + try: + rule.save_toml() + except Exception as e: + if skip_errors: + print(f'- skipping {rule.contents.data.name} - {type(e).__name__}') + errors.append(f'- {rule.contents.data.name} - {e}') + continue + raise + + saved.append(rule) + + click.echo(f'{len(results)} rules exported') + click.echo(f'{len(exported)} rules converted') + click.echo(f'{len(saved)} saved to {directory}') + if errors: + err_file = directory / '_errors.txt' + err_file.write_text('\n'.join(errors)) + click.echo(f'{len(errors)} errors saved to {err_file}') + + return exported + + @kibana_group.command('search-alerts') @click.argument('query', required=False) @click.option('--date-range', '-d', type=(str, str), default=('now-7d', 'now'), help='Date range to scope search') diff --git a/detection_rules/main.py b/detection_rules/main.py index 09758fa91..882e6278e 100644 --- a/detection_rules/main.py +++ b/detection_rules/main.py @@ -8,7 +8,6 @@ import dataclasses import glob import json import os -import re import time from datetime import datetime @@ -29,7 +28,7 @@ from .rule import TOMLRule, TOMLRuleContents, QueryRuleData from .rule_formatter import toml_write from .rule_loader import RuleCollection from .schemas import all_versions, definitions, get_incompatible_fields, get_schema_file -from .utils import Ndjson, get_path, get_etc_path, clear_caches, load_dump, load_rule_contents +from .utils import Ndjson, get_path, get_etc_path, clear_caches, load_dump, load_rule_contents, rulename_to_filename RULES_DIR = get_path('rules') @@ -92,11 +91,11 @@ def generate_rules_index(ctx: click.Context, query, overwrite, save_files=True): return bulk_upload_docs, importable_rules_docs -@root.command('import-rules') +@root.command('import-rules-to-repo') @click.argument('input-file', type=click.Path(dir_okay=False, exists=True), nargs=-1, required=False) @click.option('--required-only', is_flag=True, help='Only prompt for required fields') @click.option('--directory', '-d', type=click.Path(file_okay=False, exists=True), help='Load files from a directory') -def import_rules(input_file, required_only, directory): +def import_rules_into_repo(input_file, required_only, directory): """Import rules from json, toml, yaml, or Kibana exported rule file(s).""" rule_files = glob.glob(os.path.join(directory, '**', '*.*'), recursive=True) if directory else [] rule_files = sorted(set(rule_files + list(input_file))) @@ -108,12 +107,9 @@ def import_rules(input_file, required_only, directory): if not rule_contents: click.echo('Must specify at least one file!') - def name_to_filename(name): - return re.sub(r'[^_a-z0-9]+', '_', name.strip().lower()).strip('_') + '.toml' - for contents in rule_contents: base_path = contents.get('name') or contents.get('rule', {}).get('name') - base_path = name_to_filename(base_path) if base_path else base_path + base_path = rulename_to_filename(base_path) if base_path else base_path rule_path = os.path.join(RULES_DIR, base_path) if base_path else None additional = ['index'] if not contents.get('data_view_id') else ['data_view_id'] rule_prompt(rule_path, required_only=required_only, save=True, verbose=True, @@ -274,7 +270,7 @@ def _export_rules(rules: RuleCollection, outfile: Path, downgrade_version: Optio click.echo(f'Skipped {len(unsupported)} unsupported rules: \n- {unsupported_str}') -@root.command('export-rules') +@root.command('export-rules-from-repo') @multi_collection @click.option('--outfile', '-o', default=Path(get_path('exports', f'{time.strftime("%Y%m%dT%H%M%SL")}.ndjson')), type=Path, help='Name of file for exported rules') @@ -285,8 +281,8 @@ def _export_rules(rules: RuleCollection, outfile: Path, downgrade_version: Optio help='If `--stack-version` is passed, skip rule types which are unsupported ' '(an error will be raised otherwise)') @click.option('--include-metadata', type=bool, is_flag=True, default=False, help='Add metadata to the exported rules') -def export_rules(rules, outfile: Path, replace_id, stack_version, - skip_unsupported, include_metadata: bool) -> RuleCollection: +def export_rules_from_repo(rules, outfile: Path, replace_id, stack_version, + skip_unsupported, include_metadata: bool) -> RuleCollection: """Export rule(s) into an importable ndjson file.""" assert len(rules) > 0, "No rules found" diff --git a/detection_rules/mixins.py b/detection_rules/mixins.py index 119d87462..52cee2399 100644 --- a/detection_rules/mixins.py +++ b/detection_rules/mixins.py @@ -5,15 +5,17 @@ """Generic mixin classes.""" +import dataclasses from pathlib import Path -from typing import Any, Optional, TypeVar, Type +from typing import Any, Optional, TypeVar, Type, Literal import json import marshmallow_dataclass import marshmallow_dataclass.union_field import marshmallow_jsonschema import marshmallow_union -from marshmallow import Schema, ValidationError, fields, validates_schema +import marshmallow +from marshmallow import Schema, ValidationError, validates_schema, fields as marshmallow_fields from .misc import load_current_package_version from .schemas import definitions @@ -23,6 +25,7 @@ from .utils import cached, dict_hash T = TypeVar('T') ClassT = TypeVar('ClassT') # bound=dataclass? +UNKNOWN_VALUES = Literal['raise', 'exclude', 'include'] def _strip_none_from_dict(obj: T) -> T: @@ -81,14 +84,44 @@ def patch_jsonschema(obj: dict) -> dict: return patched +class BaseSchema(Schema): + """Base schema for marshmallow dataclasses with unknown.""" + class Meta: + """Meta class for marshmallow schema.""" + + +def exclude_class_schema( + clazz, base_schema: type[Schema] = BaseSchema, unknown: UNKNOWN_VALUES = marshmallow.EXCLUDE, **kwargs +) -> type[Schema]: + """Get a marshmallow schema for a dataclass with unknown=EXCLUDE.""" + base_schema.Meta.unknown = unknown + return marshmallow_dataclass.class_schema(clazz, base_schema=base_schema, **kwargs) + + +def recursive_class_schema( + clazz, base_schema: type[Schema] = BaseSchema, unknown: UNKNOWN_VALUES = marshmallow.EXCLUDE, **kwargs +) -> type[Schema]: + """Recursively apply the unknown parameter for nested schemas.""" + schema = exclude_class_schema(clazz, base_schema=base_schema, unknown=unknown, **kwargs) + for field in dataclasses.fields(clazz): + if dataclasses.is_dataclass(field.type): + nested_cls = field.type + nested_schema = recursive_class_schema(nested_cls, base_schema=base_schema, **kwargs) + setattr(schema, field.name, nested_schema) + return schema + + class MarshmallowDataclassMixin: """Mixin class for marshmallow serialization.""" @classmethod @cached - def __schema(cls: ClassT) -> Schema: + def __schema(cls: ClassT, unknown: Optional[UNKNOWN_VALUES] = None) -> Schema: """Get the marshmallow schema for the data class""" - return marshmallow_dataclass.class_schema(cls)() + if unknown: + return recursive_class_schema(cls, unknown=unknown)() + else: + return marshmallow_dataclass.class_schema(cls)() def get(self, key: str, default: Optional[Any] = None): """Get a key from the query data without raising attribute errors.""" @@ -103,9 +136,9 @@ class MarshmallowDataclassMixin: return jsonschema @classmethod - def from_dict(cls: Type[ClassT], obj: dict) -> ClassT: + def from_dict(cls: Type[ClassT], obj: dict, unknown: Optional[UNKNOWN_VALUES] = None) -> ClassT: """Deserialize and validate a dataclass from a dict using marshmallow.""" - schema = cls.__schema() + schema = cls.__schema(unknown=unknown) return schema.load(obj) def to_dict(self, strip_none_values=True) -> dict: @@ -199,7 +232,7 @@ class PatchedJSONSchema(marshmallow_jsonschema.JSONSchema): # Patch marshmallow-jsonschema to support marshmallow-dataclass[union] def _get_schema_for_field(self, obj, field): """Patch marshmallow_jsonschema.base.JSONSchema to support marshmallow-dataclass[union].""" - if isinstance(field, fields.Raw) and field.allow_none and not field.validate: + if isinstance(field, marshmallow_fields.Raw) and field.allow_none and not field.validate: # raw fields shouldn't be type string but type any. bug in marshmallow_dataclass:__init__.py: # if typ is Any: # metadata.setdefault("allow_none", True) diff --git a/detection_rules/rule.py b/detection_rules/rule.py index 6b4ee994e..1ac6f7a4e 100644 --- a/detection_rules/rule.py +++ b/detection_rules/rule.py @@ -7,6 +7,7 @@ import copy import dataclasses import json import os +import time import typing from abc import ABC, abstractmethod from dataclasses import dataclass, field @@ -16,6 +17,7 @@ from typing import Any, Dict, List, Literal, Optional, Tuple, Union from uuid import uuid4 import eql +import marshmallow from semver import Version from marko.block import Document as MarkoDocument from marko.ext.gfm import gfm @@ -38,6 +40,7 @@ from .utils import cached, convert_time_span, PatchedTemplate _META_SCHEMA_REQ_DEFAULTS = {} MIN_FLEET_PACKAGE_VERSION = '7.13.0' +TIME_NOW = time.strftime('%Y/%m/%d') BUILD_FIELD_VERSIONS = { "related_integrations": (Version.parse('8.3.0'), None), @@ -46,6 +49,42 @@ BUILD_FIELD_VERSIONS = { } +@dataclass +class DictRule: + """Simple object wrapper for raw rule dicts.""" + + contents: dict + path: Optional[Path] = None + + @property + def metadata(self) -> dict: + """Metadata portion of TOML file rule.""" + return self.contents.get('metadata', {}) + + @property + def data(self) -> dict: + """Rule portion of TOML file rule.""" + return self.contents.get('data') or self.contents + + @property + def id(self) -> str: + """Get the rule ID.""" + return self.data['rule_id'] + + @property + def name(self) -> str: + """Get the rule name.""" + return self.data['name'] + + def __hash__(self) -> int: + """Get the hash of the rule.""" + return hash(self.id + self.name) + + def __repr__(self) -> str: + """Get a string representation of the rule.""" + return f"Rule({self.name} {self.id})" + + @dataclass(frozen=True) class RuleMeta(MarshmallowDataclassMixin): """Data stored in a rule's [metadata] section of TOML.""" @@ -1199,6 +1238,15 @@ class TOMLRuleContents(BaseRuleContents, MarshmallowDataclassMixin): def validate_remote(remote_validator: 'RemoteValidator', contents: 'TOMLRuleContents'): remote_validator.validate_rule(contents) + @classmethod + def from_rule_resource( + cls, rule: dict, creation_date: str = TIME_NOW, updated_date: str = TIME_NOW, maturity: str = 'development' + ) -> 'TOMLRuleContents': + """Create a TOMLRuleContents from a kibana rule resource.""" + meta = {'creation_date': creation_date, 'updated_date': updated_date, 'maturity': maturity} + contents = cls.from_dict({'metadata': meta, 'rule': rule, 'transforms': None}, unknown=marshmallow.EXCLUDE) + return contents + def to_dict(self, strip_none_values=True) -> dict: # Load schemas directly from the data and metadata classes to avoid schema ambiguity which can # result from union fields which contain classes and related subclasses (AnyRuleData). See issue #1141 diff --git a/detection_rules/rule_loader.py b/detection_rules/rule_loader.py index 5e6da1737..fbd0364cf 100644 --- a/detection_rules/rule_loader.py +++ b/detection_rules/rule_loader.py @@ -13,12 +13,14 @@ from typing import Callable, Dict, Iterable, List, Optional, Union import click import pytoml +import json from marshmallow.exceptions import ValidationError from . import utils from .mappings import RtaMappings -from .rule import (DeprecatedRule, DeprecatedRuleContents, TOMLRule, - TOMLRuleContents) +from .rule import ( + DeprecatedRule, DeprecatedRuleContents, DictRule, TOMLRule, TOMLRuleContents +) from .schemas import definitions from .utils import cached, get_path @@ -153,6 +155,163 @@ class DeprecatedCollection(BaseCollection): return filtered_collection +class RawRuleCollection(BaseCollection): + """Collection of rules in raw dict form.""" + + __default = None + __default_bbr = None + + def __init__(self, rules: Optional[List[dict]] = None, ext_patterns: Optional[List[str]] = None): + """Create a new raw rule collection, with optional file ext pattern override.""" + # ndjson is unsupported since it breaks the contract of 1 rule per file, so rules should be manually broken out + # first + self.ext_patterns = ext_patterns or ['*.toml', '*.json'] + self.id_map: Dict[definitions.UUIDString, DictRule] = {} + self.file_map: Dict[Path, DictRule] = {} + self.name_map: Dict[definitions.RuleName, DictRule] = {} + self.rules: List[DictRule] = [] + self.errors: Dict[Path, Exception] = {} + self.frozen = False + + self._raw_load_cache: Dict[Path, dict] = {} + for rule in (rules or []): + self.add_rule(rule) + + def __contains__(self, rule: DictRule): + """Check if a rule is in the map by comparing IDs.""" + return rule.id in self.id_map + + def filter(self, cb: Callable[[DictRule], bool]) -> 'RawRuleCollection': + """Retrieve a filtered collection of rules.""" + filtered_collection = RawRuleCollection() + + for rule in filter(cb, self.rules): + filtered_collection.add_rule(rule) + + return filtered_collection + + def _load_rule_file(self, path: Path) -> dict: + """Load a rule file into a dictionary.""" + if path in self._raw_load_cache: + return self._raw_load_cache[path] + + if path.suffix == ".toml": + # use pytoml instead of toml because of annoying bugs + # https://github.com/uiri/toml/issues/152 + # might also be worth looking at https://github.com/sdispater/tomlkit + raw_dict = pytoml.loads(path.read_text()) + elif path.suffix == ".json": + raw_dict = json.loads(path.read_text()) + elif path.suffix == ".ndjson": + raise ValueError('ndjson is not supported in RawRuleCollection. Break out the rules individually.') + else: + raise ValueError(f"Unsupported file type {path.suffix} for rule {path}") + + self._raw_load_cache[path] = raw_dict + return raw_dict + + def _get_paths(self, directory: Path, recursive=True) -> List[Path]: + """Get all paths in a directory that match the ext patterns.""" + paths = [] + for pattern in self.ext_patterns: + paths.extend(sorted(directory.rglob(pattern) if recursive else directory.glob(pattern))) + return paths + + def _assert_new(self, rule: DictRule): + """Assert that a rule is new and can be added to the collection.""" + id_map = self.id_map + file_map = self.file_map + name_map = self.name_map + + assert not self.frozen, f"Unable to add rule {rule.name} {rule.id} to a frozen collection" + assert rule.id not in id_map, \ + f"Rule ID {rule.id} for {rule.name} collides with rule {id_map.get(rule.id).name}" + assert rule.name not in name_map, \ + f"Rule Name {rule.name} for {rule.id} collides with rule ID {name_map.get(rule.name).id}" + + if rule.path is not None: + rule_path = rule.path.resolve() + assert rule_path not in file_map, f"Rule file {rule_path} already loaded" + file_map[rule_path] = rule + + def add_rule(self, rule: DictRule): + """Add a rule to the collection.""" + self._assert_new(rule) + self.id_map[rule.id] = rule + self.name_map[rule.name] = rule + self.rules.append(rule) + + def load_dict(self, obj: dict, path: Optional[Path] = None) -> DictRule: + """Load a rule from a dictionary.""" + rule = DictRule(contents=obj, path=path) + self.add_rule(rule) + return rule + + def load_file(self, path: Path) -> DictRule: + """Load a rule from a file.""" + try: + path = path.resolve() + # use the default rule loader as a cache. + # if it already loaded the rule, then we can just use it from that + if self.__default is not None and self is not self.__default: + if path in self.__default.file_map: + rule = self.__default.file_map[path] + self.add_rule(rule) + return rule + + obj = self._load_rule_file(path) + return self.load_dict(obj, path=path) + except Exception: + print(f"Error loading rule in {path}") + raise + + def load_files(self, paths: Iterable[Path]): + """Load multiple files into the collection.""" + for path in paths: + self.load_file(path) + + def load_directory(self, directory: Path, recursive=True, obj_filter: Optional[Callable[[dict], bool]] = None): + """Load all rules in a directory.""" + paths = self._get_paths(directory, recursive=recursive) + if obj_filter is not None: + paths = [path for path in paths if obj_filter(self._load_rule_file(path))] + + self.load_files(paths) + + def load_directories(self, directories: Iterable[Path], recursive=True, + obj_filter: Optional[Callable[[dict], bool]] = None): + """Load all rules in multiple directories.""" + for path in directories: + self.load_directory(path, recursive=recursive, obj_filter=obj_filter) + + def freeze(self): + """Freeze the rule collection and make it immutable going forward.""" + self.frozen = True + + @classmethod + def default(cls) -> 'RawRuleCollection': + """Return the default rule collection, which retrieves from rules/.""" + if cls.__default is None: + collection = RawRuleCollection() + collection.load_directory(DEFAULT_RULES_DIR) + collection.load_directory(DEFAULT_BBR_DIR) + collection.freeze() + cls.__default = collection + + return cls.__default + + @classmethod + def default_bbr(cls) -> 'RawRuleCollection': + """Return the default BBR collection, which retrieves from building_block_rules/.""" + if cls.__default_bbr is None: + collection = RawRuleCollection() + collection.load_directory(DEFAULT_BBR_DIR) + collection.freeze() + cls.__default_bbr = collection + + return cls.__default_bbr + + class RuleCollection(BaseCollection): """Collection of rule objects.""" @@ -327,17 +486,17 @@ class RuleCollection(BaseCollection): for path in paths: self.load_file(path) - def load_directory(self, directory: Path, recursive=True, toml_filter: Optional[Callable[[dict], bool]] = None): + def load_directory(self, directory: Path, recursive=True, obj_filter: Optional[Callable[[dict], bool]] = None): paths = self._get_paths(directory, recursive=recursive) - if toml_filter is not None: - paths = [path for path in paths if toml_filter(self._load_toml_file(path))] + if obj_filter is not None: + paths = [path for path in paths if obj_filter(self._load_toml_file(path))] self.load_files(paths) def load_directories(self, directories: Iterable[Path], recursive=True, - toml_filter: Optional[Callable[[dict], bool]] = None): + obj_filter: Optional[Callable[[dict], bool]] = None): for path in directories: - self.load_directory(path, recursive=recursive, toml_filter=toml_filter) + self.load_directory(path, recursive=recursive, obj_filter=obj_filter) def freeze(self): """Freeze the rule collection and make it immutable going forward.""" @@ -471,9 +630,11 @@ rta_mappings = RtaMappings() __all__ = ( "FILE_PATTERN", "DEFAULT_RULES_DIR", + "DEFAULT_BBR_DIR", "load_github_pr_rules", "DeprecatedCollection", "DeprecatedRule", + "RawRuleCollection", "RuleCollection", "metadata_filter", "production_filter", diff --git a/detection_rules/utils.py b/detection_rules/utils.py index 6bc7e527f..abb5c011a 100644 --- a/detection_rules/utils.py +++ b/detection_rules/utils.py @@ -13,6 +13,7 @@ import hashlib import io import json import os +import re import shutil import subprocess import zipfile @@ -306,6 +307,15 @@ def clear_caches(): _cache.clear() +def rulename_to_filename(name: str, tactic_name: str = None, ext: str = '.toml') -> str: + """Convert a rule name to a filename.""" + name = re.sub(r'[^_a-z0-9]+', '_', name.strip().lower()).strip('_') + if tactic_name: + pre = rulename_to_filename(name=tactic_name, ext='') + name = f'{pre}_{name}' + return name + ext or '' + + def load_rule_contents(rule_file: Path, single_only=False) -> list: """Load a rule file from multiple formats.""" _, extension = os.path.splitext(rule_file) diff --git a/docs/developing.md b/docs/developing.md index b7b57ba9c..b47eb0104 100644 --- a/docs/developing.md +++ b/docs/developing.md @@ -36,3 +36,27 @@ relativeTo = "now" Other transform suppoprt can be found under `python -m detection-rules dev transforms -h` + + +## Using the `RuleResource` methods built on detections `_bulk_action` APIs + +The following is meant to serve as a simple example of to use the methods + +```python +import kibana +from kibana import definitions + +rids = ['40e1f208-aaaa-bbbb-98ea-378ccf504ad3', '5e9bc07c-cccc-dddd-a6c0-1cae4a0d256e'] + +# with TypedDict, either is valid, both with static type checking +set_tags = definitions.RuleBulkSetTags(type='set_tags', value=['tag1', 'tag2']) +delete_tags: definitions.RuleBulkDeleteTags = {'type': 'delete_tags', 'value': ['tag1', 'tag2']} + +with kibana: + r1 = RuleResource.bulk_enable(rids, dry_run=True) + r2 = RuleResource.bulk_disable(rids, dry_run=True) + r3 = RuleResource.bulk_duplicate(rids, dry_run=True) + r4 = RuleResource.bulk_export(rids) + r5 = RuleResource.bulk_edit(edit_object=[set_tags, delete_tags], rule_ids=rids, dry_run=True) + r6 = RuleResource.bulk_delete(rids, dry_run=True) +``` diff --git a/lib/kibana/kibana/__init__.py b/lib/kibana/kibana/__init__.py index d96615c4c..45731b5e1 100644 --- a/lib/kibana/kibana/__init__.py +++ b/lib/kibana/kibana/__init__.py @@ -8,7 +8,7 @@ from .connector import Kibana from .resources import RuleResource, Signal -__version__ = '0.1.0' +__version__ = '0.2.0' __all__ = ( "Kibana", "RuleResource", diff --git a/lib/kibana/kibana/connector.py b/lib/kibana/kibana/connector.py index ff1275316..29ad8d606 100644 --- a/lib/kibana/kibana/connector.py +++ b/lib/kibana/kibana/connector.py @@ -10,6 +10,7 @@ import json import sys import threading import uuid +from typing import List, Optional, Union import requests from elasticsearch import Elasticsearch @@ -72,6 +73,16 @@ class Kibana(object): if self.status: return self.status.get("version", {}).get("number") + @staticmethod + def ndjson_file_data_prep(lines: List[dict], filename: str) -> (dict, str): + """Prepare a request for an ndjson file upload to Kibana.""" + data = ('\n'.join(json.dumps(r) for r in lines) + '\n') + boundary = '----JustAnotherBoundary' + bounded_data = (f'--{boundary}\r\nContent-Disposition: form-data; name="file"; filename="{filename}"\r\n' + f'Content-Type: application/x-ndjson\r\n\r\n{data}\r\n--{boundary}--\r\n').encode('utf-8') + headers = {'content-type': f'multipart/form-data; boundary={boundary}'} + return headers, bounded_data + def url(self, uri): """Get the full URL given a URI.""" assert self.kibana_url is not None @@ -81,14 +92,15 @@ class Kibana(object): uri = "s/{}/{}".format(self.space, uri) return f"{self.kibana_url}/{uri}" - def request(self, method, uri, params=None, data=None, error=True, verbose=True, raw=False, **kwargs): + def request(self, method, uri, params=None, data=None, raw_data=None, error=True, verbose=True, raw=False, + **kwargs) -> Optional[Union[requests.Response, dict]]: """Perform a RESTful HTTP request with JSON responses.""" - params = params or {} url = self.url(uri) - params = {k: v for k, v in params.items()} - body = None - if data is not None: - body = json.dumps(data) + params = params or {} + body = json.dumps(data) if data is not None else None + assert not (body and raw_data), "Cannot provide both data and raw_data" + + body = body or raw_data response = self.session.request(method, url, params=params, data=body, **kwargs) @@ -100,6 +112,8 @@ class Kibana(object): try: response.raise_for_status() except requests.exceptions.HTTPError: + if response.status_code == 404: + raise NotImplementedError(f'API endpoint {uri} not implemented for Kibana version {self.version}') if verbose: print(response.content.decode("utf-8"), file=sys.stderr) raise @@ -107,7 +121,7 @@ class Kibana(object): if not response.content: return - return response.content if raw else response.json() + return response if raw else response.json() def get(self, uri, params=None, data=None, error=True, **kwargs): """Perform an HTTP GET.""" diff --git a/lib/kibana/kibana/definitions.py b/lib/kibana/kibana/definitions.py new file mode 100644 index 000000000..87936a825 --- /dev/null +++ b/lib/kibana/kibana/definitions.py @@ -0,0 +1,88 @@ +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0; you may not use this file except in compliance with the Elastic License +# 2.0. + +from typing import Literal, Union, TypedDict, NotRequired + + +RuleBulkActions = Literal['enable', 'disable', 'delete', 'duplicate', 'export', 'edit'] + + +class RuleBulkAddTags(TypedDict): + type: Literal['add_tags'] + value: list[str] + + +class RuleBulkDeleteTags(TypedDict): + type: Literal['delete_tags'] + value: list[str] + + +class RuleBulkSetTags(TypedDict): + type: Literal['set_tags'] + value: list[str] + + +class RuleBulkAddIndexPatterns(TypedDict): + type: Literal['add_index_patterns'] + value: list[str] + + +class RuleBulkDeleteIndexPatterns(TypedDict): + type: Literal['delete_index_patterns'] + value: list[str] + + +class RuleBulkSetIndexPatterns(TypedDict): + type: Literal['set_index_patterns'] + value: list[str] + + +class _ValueSetTimeline(TypedDict): + timeline_id: str + timeline_title: str + + +class RuleBulkSetTimeline(TypedDict): + type: Literal['set_timeline'] + value: _ValueSetTimeline + + +class _ValueSetSchedule(TypedDict): + interval: str + lookback: str + + +class RuleBulkSetSchedule(TypedDict): + type: Literal['set_schedule'] + value: _ValueSetSchedule + + +class _ValueAddOrSetRuleActions(TypedDict): + actions: list[dict] # intentionally not setting based on literal values + throttle: NotRequired[dict] # to be deprecated + + +class RuleBulkAddRuleActions(TypedDict): + type: Literal['add_rule_actions'] + value: _ValueAddOrSetRuleActions + + +class RuleBulkSetRuleActions(TypedDict): + type: Literal['set_rule_actions'] + value: _ValueAddOrSetRuleActions + + +RuleBulkEditActionTypes = Union[ + RuleBulkAddTags, + RuleBulkDeleteTags, + RuleBulkSetTags, + RuleBulkAddIndexPatterns, + RuleBulkDeleteIndexPatterns, + RuleBulkSetIndexPatterns, + RuleBulkSetTimeline, + RuleBulkSetSchedule, + RuleBulkAddRuleActions, + RuleBulkSetRuleActions +] diff --git a/lib/kibana/kibana/resources.py b/lib/kibana/kibana/resources.py index 8855bdfe1..c056d4235 100644 --- a/lib/kibana/kibana/resources.py +++ b/lib/kibana/kibana/resources.py @@ -4,9 +4,12 @@ # 2.0. import datetime -from typing import List, Optional, Type +from typing import Any, List, Optional, Type + +import json from .connector import Kibana +from . import definitions DEFAULT_PAGE_SIZE = 10 @@ -20,10 +23,12 @@ class BaseResource(dict): return self.get(self.ID_FIELD) @classmethod - def bulk_create(cls, resources: list): + def bulk_create_legacy(cls, resources: list): for r in resources: assert isinstance(r, cls) + # _bulk_create is being deprecated. Leave for backwards compat only + # the new API would be import with multiple rules within an ndjson request responses = Kibana.current().post(cls.BASE_URI + "/_bulk_create", data=resources) return [cls(r) for r in responses] @@ -127,6 +132,88 @@ class RuleResource(BaseResource): params = cls._add_internal_filter(True, params) return cls.find(**params) + @classmethod + def bulk_action( + cls, action: definitions.RuleBulkActions, rule_ids: Optional[List[str]] = None, query: Optional[str] = None, + dry_run: Optional[bool] = False, edit_object: Optional[list[definitions.RuleBulkEditActionTypes]] = None, + include_exceptions: Optional[bool] = False, **kwargs + ) -> (dict, List['RuleResource']): + """Perform a bulk action on rules using the _bulk_action API.""" + assert not (rule_ids and query), 'Cannot provide both rule_ids and query' + + if action == 'edit': + assert edit_object, 'edit action requires edit object' + + duplicate = {'include_exceptions': include_exceptions, 'include_expired_exceptions': False} + + params = dict(dry_run=stringify_bool(dry_run)) + data = dict(action=action, edit=edit_object, duplicate=duplicate) + if query: + data['query'] = query + elif rule_ids: + data['rule_ids'] = rule_ids + response = Kibana.current().post(cls.BASE_URI + "/_bulk_action", params=params, data=data, **kwargs) + + # export returns ndjson, which requires manual parsing since response.json() fails + if action == 'export': + response = [json.loads(r) for r in response.text.splitlines()] + result_ids = [r['rule_id'] for r in response if 'rule_id' in r] + else: + results = response['attributes']['results'] + result_ids = [r['rule_id'] for r in results['updated']] + result_ids.extend([r['rule_id'] for r in results['created']]) + + rule_resources = cls.export_rules(result_ids) + return response, rule_resources + + @classmethod + def bulk_enable( + cls, rule_ids: Optional[List[str]] = None, query: Optional[str] = None, dry_run: Optional[bool] = False + ) -> (dict, List['RuleResource']): + """Bulk enable rules using _bulk_action.""" + return cls.bulk_action("enable", rule_ids=rule_ids, query=query, dry_run=dry_run) + + @classmethod + def bulk_disable( + cls, rule_ids: Optional[List[str]] = None, query: Optional[str] = None, dry_run: Optional[bool] = False + ) -> (dict, List['RuleResource']): + """Bulk disable rules using _bulk_action.""" + return cls.bulk_action("disable", rule_ids=rule_ids, query=query, dry_run=dry_run) + + @classmethod + def bulk_delete( + cls, rule_ids: Optional[List[str]] = None, query: Optional[str] = None, dry_run: Optional[bool] = False + ) -> (dict, List['RuleResource']): + """Bulk delete rules using _bulk_action.""" + return cls.bulk_action("delete", rule_ids=rule_ids, query=query, dry_run=dry_run) + + @classmethod + def bulk_duplicate( + cls, rule_ids: Optional[List[str]] = None, query: Optional[str] = None, dry_run: Optional[bool] = False, + include_exceptions: Optional[bool] = False + ) -> (dict, List['RuleResource']): + """Bulk duplicate rules using _bulk_action.""" + return cls.bulk_action("duplicate", rule_ids=rule_ids, query=query, dry_run=dry_run, + include_exceptions=include_exceptions) + + @classmethod + def bulk_export( + cls, rule_ids: Optional[List[str]] = None, query: Optional[str] = None + ) -> (dict, List['RuleResource']): + """Bulk export rules using _bulk_action.""" + return cls.bulk_action("export", rule_ids=rule_ids, query=query, raw=True) + + @classmethod + def bulk_edit( + cls, edit_object: list[definitions.RuleBulkEditActionTypes], rule_ids: Optional[List[str]] = None, + query: Optional[str] = None, dry_run: Optional[bool] = False + ) -> (dict, List['RuleResource']): + """Bulk edit rules using _bulk_action.""" + # setting to error=False because the API returns a 500 with any failures, but includes the success data as well + return cls.bulk_action( + "edit", rule_ids=rule_ids, query=query, dry_run=dry_run, edit_object=edit_object, error=False + ) + def put(self): # id and rule_id are mutually exclusive rule_id = self.get("rule_id") @@ -142,6 +229,43 @@ class RuleResource(BaseResource): raise + @classmethod + def import_rules(cls, rules: List[dict], overwrite: bool = False, overwrite_exceptions: bool = False, + overwrite_action_connectors: bool = False) -> (dict, list, List[Optional['RuleResource']]): + """Import a list of rules into Kibana using the _import API and return the response and successful imports.""" + url = f'{cls.BASE_URI}/_import' + params = dict( + overwrite=stringify_bool(overwrite), + overwrite_exceptions=stringify_bool(overwrite_exceptions), + overwrite_action_connectors=stringify_bool(overwrite_action_connectors) + ) + rule_ids = [r['rule_id'] for r in rules] + headers, raw_data = Kibana.ndjson_file_data_prep(rules, "import.ndjson") + response = Kibana.current().post(url, headers=headers, params=params, raw_data=raw_data) + errors = response.get("errors", []) + error_rule_ids = [e['rule_id'] for e in errors] + + # successful rule_ids are not returned, so they must be implicitly inferred from errored rule_ids + successful_rule_ids = [r for r in rule_ids if r not in error_rule_ids] + rule_resources = cls.export_rules(successful_rule_ids) if successful_rule_ids else [] + return response, successful_rule_ids, rule_resources + + @classmethod + def export_rules(cls, rule_ids: Optional[List[str]] = None, + exclude_export_details: bool = True) -> List['RuleResource']: + """Export a list of rules from Kibana using the _export API.""" + url = f'{cls.BASE_URI}/_export' + + if rule_ids: + rule_ids = {'objects': [{'rule_id': r} for r in rule_ids]} + else: + rule_ids = None + + params = dict(exclude_export_details=stringify_bool(exclude_export_details)) + response = Kibana.current().post(url, params=params, data=rule_ids, raw=True) + data = [json.loads(r) for r in response.text.splitlines()] + return [cls(r) for r in data] + class Signal(BaseResource): BASE_URI = "/api/detection_engine/signals" @@ -194,3 +318,9 @@ class Signal(BaseResource): @classmethod def open_many(cls, signal_ids: List[str]): return cls.set_status_many(signal_ids, "open") + + +def stringify_bool(obj: bool) -> str: + """Convert a boolean to a string.""" + assert isinstance(obj, bool), f"Expected a boolean, got {type(obj)}" + return str(obj).lower() diff --git a/lib/kibana/pyproject.toml b/lib/kibana/pyproject.toml index 6565fe529..b577a5100 100644 --- a/lib/kibana/pyproject.toml +++ b/lib/kibana/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "detection-rules-kibana" -version = "0.1.0" +version = "0.2.0" description = "Kibana API utilities for Elastic Detection Rules" license = {text = "Elastic License v2"} keywords = ["Elastic", "Kibana", "Detection Rules", "Security", "Elasticsearch"]