From eadcd9d3e0f6ba2797303354626c4d4fe8c7d667 Mon Sep 17 00:00:00 2001 From: Eric Forte <119343520+eric-forte-elastic@users.noreply.github.com> Date: Mon, 10 Mar 2025 12:59:12 -0400 Subject: [PATCH] [FR] Add Env Var DR_CLI_MAX_WIDTH and DaC Docs Updates (#4518) * Add Env Var DR_CLI_MAX_WIDTH * Version Bump * Update limit from 120 to 240 * Clean references to reference main * Update Readme with DaC Info * Add DaC to Table of Contents * Bump Patch Version * Updated naming and add dac md * Organize Imports * Deprecate upload-rule * Update docs/detections-as-code.md Co-authored-by: Mika Ayenson, PhD * move docs to docs-dev * Sort custom rules imports * Remove duplicate * Fix typo * Bump Patch Version --------- Co-authored-by: Mika Ayenson, PhD --- CLI.md | 219 ++++++++---------- README.md | 6 + detection_rules/custom_rules.py | 9 +- detection_rules/kbwrap.py | 7 +- detection_rules/main.py | 8 +- detection_rules/ml.py | 2 +- ...om-rules.md => custom-rules-management.md} | 0 docs-dev/detections-as-code.md | 57 +++++ .../experimental-detections.md | 2 +- pyproject.toml | 2 +- 10 files changed, 183 insertions(+), 129 deletions(-) rename docs-dev/{custom-rules.md => custom-rules-management.md} (100%) create mode 100644 docs-dev/detections-as-code.md diff --git a/CLI.md b/CLI.md index 98f888611..81da5ea51 100644 --- a/CLI.md +++ b/CLI.md @@ -44,6 +44,9 @@ Using the environment variable `DR_BYPASS_TAGS_VALIDATION` will bypass the Detec Using the environment variable `DR_BYPASS_TIMELINE_TEMPLATE_VALIDATION` will bypass the timeline template id and title validation for rules. +Using the environment variable `DR_CLI_MAX_WIDTH` will set a custom max width for the click CLI. +For instance, some users may want to increase the default value in cases where help messages are cut off. + ## Importing rules into the repo You can import rules into the repo using the `create-rule` or `import-rules-to-repo` commands. Both of these commands will @@ -127,28 +130,6 @@ Commands which connect to Elasticsearch or Kibana are embedded under the subcomm These command groups will leverage their respective clients and will automatically use parsed config options if defined, otherwise arguments should be passed to the sub-command as: -`python -m detection-rules kibana -u -p upload-rule <...>` - - -```console -python -m detection_rules es -h - -Usage: detection_rules es [OPTIONS] COMMAND [ARGS]... - - Commands for integrating with Elasticsearch. - -Options: - -et, --timeout INTEGER Timeout for elasticsearch client - -ep, --es-password TEXT - -eu, --es-user TEXT - --cloud-id TEXT - -e, --elasticsearch-url TEXT - -h, --help Show this message and exit. - -Commands: - collect-events Collect events from Elasticsearch. -``` - Providers are the name that Elastic Cloud uses to configure authentication in Kibana. When we create deployment, Elastic Cloud configures two providers by default: basic/cloud-basic and saml/cloud-saml (for SSO). ```console @@ -165,26 +146,21 @@ Usage: detection_rules kibana [OPTIONS] COMMAND [ARGS]... Options: --ignore-ssl-errors TEXT --space TEXT Kibana space - --provider-name TEXT For cloud deployments, Elastic Cloud configures - two providers by default: cloud-basic and - cloud-saml (for SSO) - --provider-type TEXT For cloud deployments, Elastic Cloud configures - two providers by default: basic and saml (for - SSO) + --provider-name TEXT Elastic Cloud providers: cloud-basic and cloud-saml (for SSO) + --provider-type TEXT Elastic Cloud providers: basic and saml (for SSO) -ku, --kibana-user TEXT --kibana-url TEXT -kp, --kibana-password TEXT -kc, --kibana-cookie TEXT Cookie from an authed session - --cloud-id TEXT ID of the cloud instance. Defaults the cloud - provider to cloud-basic if this option is - supplied + --api-key TEXT + --cloud-id TEXT ID of the cloud instance. -h, --help Show this message and exit. Commands: export-rules Export custom rules from Kibana. import-rules Import custom rules into Kibana. search-alerts Search detection engine alerts with KQL. - upload-rule Upload a list of rule .toml files to Kibana. + upload-rule [Deprecated] Upload a list of rule .toml files to Kibana. ``` ## Searching Kibana for Alerts @@ -195,23 +171,22 @@ Alerts stored in Kibana can be quickly be identified by searching with the `sear ```console python -m detection_rules kibana search-alerts -h +█▀▀▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄ ▄ █▀▀▄ ▄ ▄ ▄ ▄▄▄ ▄▄▄ +█ █ █▄▄ █ █▄▄ █ █ █ █ █ █▀▄ █ █▄▄▀ █ █ █ █▄▄ █▄▄ +█▄▄▀ █▄▄ █ █▄▄ █▄▄ █ ▄█▄ █▄█ █ ▀▄█ █ ▀▄ █▄▄█ █▄▄ █▄▄ ▄▄█ + Kibana client: Options: --ignore-ssl-errors TEXT --space TEXT Kibana space - --provider-name TEXT For cloud deployments, Elastic Cloud configures - two providers by default: cloud-basic and - cloud-saml (for SSO) - --provider-type TEXT For cloud deployments, Elastic Cloud configures - two providers by default: basic and saml (for - SSO) + --provider-name TEXT Elastic Cloud providers: cloud-basic and cloud-saml (for SSO) + --provider-type TEXT Elastic Cloud providers: basic and saml (for SSO) -ku, --kibana-user TEXT --kibana-url TEXT -kp, --kibana-password TEXT -kc, --kibana-cookie TEXT Cookie from an authed session - --cloud-id TEXT ID of the cloud instance. Defaults the cloud - provider to cloud-basic if this option is - supplied + --api-key TEXT + --cloud-id TEXT ID of the cloud instance. Usage: detection_rules kibana search-alerts [OPTIONS] [QUERY] @@ -221,8 +196,8 @@ Options: -d, --date-range ... Date range to scope search -c, --columns TEXT Columns to display in table - -e, --extend If columns are specified, extend the - original columns + -e, --extend If columns are specified, extend the original columns + -m, --max-count INTEGER The max number of alerts to return -h, --help Show this message and exit. ``` @@ -254,70 +229,9 @@ Running the following command will print out a table showing any alerts that hav ``` ## Uploading rules to Kibana -Toml formatted rule files can be uploaded as custom rules using the `kibana upload-rule` command. To upload more than one -file, specify multiple files at a time as individual args. This command is meant to support uploading and testing of -rules and is not intended for production use in its current state. +### Using `kibana import-rules` -This command is built on soon to be deprecated APIs and so should be phased off. For a better option, see below... - -```console -python -m detection_rules kibana upload-rule -h - -Kibana client: -Options: - --space TEXT Kibana space - -kp, --kibana-password TEXT - -ku, --kibana-user TEXT - --cloud-id TEXT - -k, --kibana-url TEXT - -Usage: detection_rules kibana upload-rule [OPTIONS] - - Upload a list of rule .toml files to Kibana. - -Options: - -f, --rule-file FILE - -d, --directory DIRECTORY Recursively export rules from a directory - -id, --rule-id TEXT - -r, --replace-id Replace rule IDs with new IDs before export - -h, --help Show this message and exit. -(detection-rules-build) (base) ➜ detection-rules git:(rule-loader) ✗ -``` - -Alternatively, rules can be exported into a consolidated ndjson file which can be imported in the Kibana security app -directly. - -```console -Usage: detection_rules export-rules-from-repo [OPTIONS] - - Export rule(s) and exception(s) into an importable ndjson file. - -Options: - -f, --rule-file FILE - -d, --directory DIRECTORY Recursively load rules from a directory - -id, --rule-id TEXT - -o, --outfile PATH Name of file for exported rules - -r, --replace-id Replace rule IDs with new IDs before export - --stack-version [7.8|7.9|7.10|7.11|7.12|7.13|7.14|7.15|7.16|8.0|8.1|8.2|8.3|8.4|8.5|8.6|8.7|8.8|8.9|8.10|8.11|8.12|8.13|8.14] - Downgrade a rule version to be compatible - with older instances of Kibana - -s, --skip-unsupported If `--stack-version` is passed, skip rule - types which are unsupported (an error will - be raised otherwise) - --include-metadata Add metadata to the exported rules - -ac, --include-action-connectors - Include Action Connectors in export - -e, --include-exceptions Include Exceptions Lists in export - -h, --help Show this message and exit. -``` - -_*To load a custom rule, the proper index must be setup first. The simplest way to do this is to click -the `Load prebuilt detection rules and timeline templates` button on the `detections` page in the Kibana security app._ - - -### Using `import-rules` - -This is a better option than `upload-rule` as it is built on refreshed APIs +To directly load Toml formatted rule files into Kibana, one can use the `kibana import-rules` command as shown below. ``` python -m detection_rules kibana import-rules -h @@ -330,10 +244,8 @@ Kibana client: Options: --ignore-ssl-errors TEXT --space TEXT Kibana space - --provider-name TEXT Elastic Cloud providers: cloud-basic and cloud- - saml (for SSO) - --provider-type TEXT Elastic Cloud providers: basic and saml (for - SSO) + --provider-name TEXT Elastic Cloud providers: cloud-basic and cloud-saml (for SSO) + --provider-type TEXT Elastic Cloud providers: basic and saml (for SSO) -ku, --kibana-user TEXT --kibana-url TEXT -kp, --kibana-password TEXT @@ -352,8 +264,7 @@ Options: -o, --overwrite Overwrite existing rules -e, --overwrite-exceptions Overwrite exceptions in existing rules -ac, --overwrite-action-connectors - Overwrite action connectors in existing - rules + Overwrite action connectors in existing rules -h, --help Show this message and exit. ``` @@ -496,6 +407,78 @@ python -m detection_rules kibana import-rules -d test-export-rules -o +### Using `export-rules-from-repo` + +Toml formatted rule files can also be imported into Kibana through Kibana security app via a consolidated ndjson file +which is exported from detection rules. + +```console +Usage: detection_rules export-rules-from-repo [OPTIONS] + + Export rule(s) and exception(s) into an importable ndjson file. + +Options: + -f, --rule-file FILE + -d, --directory DIRECTORY Recursively load rules from a directory + -id, --rule-id TEXT + -o, --outfile PATH Name of file for exported rules + -r, --replace-id Replace rule IDs with new IDs before export + --stack-version [7.8|7.9|7.10|7.11|7.12|7.13|7.14|7.15|7.16|8.0|8.1|8.2|8.3|8.4|8.5|8.6|8.7|8.8|8.9|8.10|8.11|8.12|8.13|8.14] + Downgrade a rule version to be compatible + with older instances of Kibana + -s, --skip-unsupported If `--stack-version` is passed, skip rule + types which are unsupported (an error will + be raised otherwise) + --include-metadata Add metadata to the exported rules + -ac, --include-action-connectors + Include Action Connectors in export + -e, --include-exceptions Include Exceptions Lists in export + -h, --help Show this message and exit. +``` + +_*To load a custom rule, the proper index must be setup first. The simplest way to do this is to click +the `Load prebuilt detection rules and timeline templates` button on the `detections` page in the Kibana security app._ + + +### Deprecated Methods + +Toml formatted rule files can also be uploaded as custom rules using the `kibana upload-rule` command. This command is +deprecated as of Elastic Stack version 9.0, but is included for compatibility with older stacks. To upload more than one +file, specify multiple files at a time as individual args. This command is meant to support uploading and testing of +rules and is not intended for production use in its current state. + +```console +python -m detection_rules kibana upload-rule -h + +█▀▀▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄ ▄ █▀▀▄ ▄ ▄ ▄ ▄▄▄ ▄▄▄ +█ █ █▄▄ █ █▄▄ █ █ █ █ █ █▀▄ █ █▄▄▀ █ █ █ █▄▄ █▄▄ +█▄▄▀ █▄▄ █ █▄▄ █▄▄ █ ▄█▄ █▄█ █ ▀▄█ █ ▀▄ █▄▄█ █▄▄ █▄▄ ▄▄█ + +Kibana client: +Options: + --ignore-ssl-errors TEXT + --space TEXT Kibana space + --provider-name TEXT Elastic Cloud providers: cloud-basic and cloud-saml (for SSO) + --provider-type TEXT Elastic Cloud providers: basic and saml (for SSO) + -ku, --kibana-user TEXT + --kibana-url TEXT + -kp, --kibana-password TEXT + -kc, --kibana-cookie TEXT Cookie from an authed session + --api-key TEXT + --cloud-id TEXT ID of the cloud instance. + +Usage: detection_rules kibana upload-rule [OPTIONS] + + [Deprecated] Upload a list of rule .toml files to Kibana. + +Options: + -f, --rule-file FILE + -d, --directory DIRECTORY Recursively load rules from a directory + -id, --rule-id TEXT + -r, --replace-id Replace rule IDs with new IDs before export + -h, --help Show this message and exit. +``` + ### Exporting rules This command should be run with the `CUSTOM_RULES_DIR` envvar set, that way proper validation is applied to versioning when the rules are downloaded. See the [custom rules docs](docs-dev/custom-rules.md) for more information. @@ -511,10 +494,8 @@ Kibana client: Options: --ignore-ssl-errors TEXT --space TEXT Kibana space - --provider-name TEXT Elastic Cloud providers: cloud-basic and cloud- - saml (for SSO) - --provider-type TEXT Elastic Cloud providers: basic and saml (for - SSO) + --provider-name TEXT Elastic Cloud providers: cloud-basic and cloud-saml (for SSO) + --provider-type TEXT Elastic Cloud providers: basic and saml (for SSO) -ku, --kibana-user TEXT --kibana-url TEXT -kp, --kibana-password TEXT @@ -621,7 +602,7 @@ Unknown field data_stream.dataset:osquery_manager.result and osquery_meta.counter>0 and osquery_meta.type:diff and osquery.last_run_code:0 and osquery_meta.action:removed ^^^^^^^^^^^^^^^^^ stack: 8.9.0, beats: 8.9.0, ecs: 8.9.0 -- name - {'rule': [ValidationError({'type': ['Must be equal to eql.'], 'language': ['Must be equal to eql.']}), ValidationError({'type': ['Must be equal to esql.'], 'language': ['Must be equal to esql.']}), ValidationError({'type': ['Must be equal to threshold.'], 'threshold': ['Missing data for required field.']}), ValidationError({'type': ['Must be equal to threat_match.'], 'threat_mapping': ['Missing data for required field.'], 'threat_index': ['Missing data for required field.']}), ValidationError({'type': ['Must be equal to machine_learning.'], 'anomaly_threshold': ['Missing data for required field.'], 'machine_learning_job_id': ['Missing data for required field.']}), ValidationError({'type': ['Must be equal to query.']}), ValidationError({'new_terms': ['Missing data for required field.']})]}(venv312) ➜ detection-rules-fork git:(refresh-kibana-module-with-new-APIs) ✗ +- name - {'rule': [ValidationError({'type': ['Must be equal to eql.'], 'language': ['Must be equal to eql.']}), ValidationError({'type': ['Must be equal to esql.'], 'language': ['Must be equal to esql.']}), ValidationError({'type': ['Must be equal to threshold.'], 'threshold': ['Missing data for required field.']}), ValidationError({'type': ['Must be equal to threat_match.'], 'threat_mapping': ['Missing data for required field.'], 'threat_index': ['Missing data for required field.']}), ValidationError({'type': ['Must be equal to machine_learning.'], 'anomaly_threshold': ['Missing data for required field.'], 'machine_learning_job_id': ['Missing data for required field.']}), ValidationError({'type': ['Must be equal to query.']}), ValidationError({'new_terms': ['Missing data for required field.']})]}(venv312) ➜ detection-rules-fork git:(main) ✗ ``` @@ -670,7 +651,7 @@ method specific to the transform type. This applies to osquery and insights for the moment but could expand in the future. ``` -(venv38) ➜ detection-rules-fork git:(2597-validate-osquery-insights) python -m detection_rules dev transforms -h +(venv312) ➜ detection-rules-fork git:(main) python -m detection_rules dev transforms -h █▀▀▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄ ▄ █▀▀▄ ▄ ▄ ▄ ▄▄▄ ▄▄▄ █ █ █▄▄ █ █▄▄ █ █ █ █ █ █▀▄ █ █▄▄▀ █ █ █ █▄▄ █▄▄ @@ -692,7 +673,7 @@ Commands: ``` -(venv38) ➜ detection-rules-fork git:(2597-validate-osquery-insights) python -m detection_rules dev transforms guide-plugin-convert +(venv312) ➜ detection-rules-fork git:(main) python -m detection_rules dev transforms guide-plugin-convert █▀▀▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄ ▄ █▀▀▄ ▄ ▄ ▄ ▄▄▄ ▄▄▄ █ █ █▄▄ █ █▄▄ █ █ █ █ █ █▀▄ █ █▄▄▀ █ █ █ █▄▄ █▄▄ diff --git a/README.md b/README.md index a577d87ca..501f1875b 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ This repository was first announced on Elastic's blog post, [Elastic Security op - [Overview of this repository](#overview-of-this-repository) - [Getting started](#getting-started) - [How to contribute](#how-to-contribute) + - [Detections as Code (DaC)](#detections-as-code-dac) - [RTAs](#rtas) - [Licensing](#licensing) - [Questions? Problems? Suggestions?](#questions-problems-suggestions) @@ -133,11 +134,16 @@ For more advanced command line interface (CLI) usage, refer to the [CLI guide](C We welcome your contributions to Detection Rules! Before contributing, please familiarize yourself with this repository, its [directory structure](#overview-of-this-repository), and our [philosophy](PHILOSOPHY.md) about rule creation. When you're ready to contribute, read the [contribution guide](CONTRIBUTING.md) to learn how we turn detection ideas into production rules and validate with testing. +## Detections as Code (DaC) + +The Detection Rules repo includes a number of commands to help one manage rules with an "as code" philosophy. We recommend starting with our [DaC Specific Documentation](https://dac-reference.readthedocs.io/en/latest/) for strategies and recommended setup information. However, if you would prefer to jump right in, please see our [custom rules documentation](docs-dev/custom-rules-management.md) for information on how to configure this repo for use with custom rules followed by our [CLI documentation](CLI.md) for information on our commands to import and export rules. + ## RTAs Red Team Automations (RTAs) used to emulate attacker techniques and verify the rules can be found in dedicated repository - [Cortado](https://github.com/elastic/cortado). + ## Licensing Everything in this repository — rules, code, etc. — is licensed under the [Elastic License v2](LICENSE.txt). These rules are designed to be used in the context of the Detection Engine within the Elastic Security application. If you’re using our [Elastic Cloud managed service](https://www.elastic.co/cloud/) or the default distribution of the Elastic Stack software that includes the [full set of free features](https://www.elastic.co/subscriptions), you’ll get the latest rules the first time you navigate to the detection engine. diff --git a/detection_rules/custom_rules.py b/detection_rules/custom_rules.py index 864b2f59d..befac07ea 100644 --- a/detection_rules/custom_rules.py +++ b/detection_rules/custom_rules.py @@ -8,13 +8,12 @@ from pathlib import Path import click import yaml - -from .main import root -from .utils import get_etc_path, load_etc_dump, ROOT_DIR -from .docs import REPO_DOCS_DIR - from semver import Version +from .docs import REPO_DOCS_DIR +from .main import root +from .utils import ROOT_DIR, get_etc_path, load_etc_dump + DEFAULT_CONFIG_PATH = Path(get_etc_path('_config.yaml')) CUSTOM_RULES_DOC_PATH = Path(ROOT_DIR).joinpath(REPO_DOCS_DIR, 'custom-rules.md') diff --git a/detection_rules/kbwrap.py b/detection_rules/kbwrap.py index 8307f3a60..bd3b14b75 100644 --- a/detection_rules/kbwrap.py +++ b/detection_rules/kbwrap.py @@ -51,10 +51,15 @@ def kibana_group(ctx: click.Context, **kibana_kwargs): @click.option('--replace-id', '-r', is_flag=True, help='Replace rule IDs with new IDs before export') @click.pass_context def upload_rule(ctx, rules: RuleCollection, replace_id): - """Upload a list of rule .toml files to Kibana.""" + """[Deprecated] Upload a list of rule .toml files to Kibana.""" kibana = ctx.obj['kibana'] api_payloads = [] + click.secho( + "WARNING: This command is deprecated as of Elastic Stack version 9.0. Please use `kibana import-rules`.", + fg="yellow", + ) + for rule in rules: try: payload = downgrade_contents_from_rule(rule, kibana.version, replace_id=replace_id) diff --git a/detection_rules/main.py b/detection_rules/main.py index 9f2328f6e..b07e4c776 100644 --- a/detection_rules/main.py +++ b/detection_rules/main.py @@ -40,7 +40,13 @@ RULES_CONFIG = parse_rules_config() RULES_DIRS = RULES_CONFIG.rule_dirs -@click.group('detection-rules', context_settings={'help_option_names': ['-h', '--help']}) +@click.group( + 'detection-rules', + context_settings={ + 'help_option_names': ['-h', '--help'], + 'max_content_width': int(os.getenv('DR_CLI_MAX_WIDTH', 240)), + }, +) @click.option('--debug/--no-debug', '-D/-N', is_flag=True, default=None, help='Print full exception stacktrace on errors') @click.pass_context diff --git a/detection_rules/ml.py b/detection_rules/ml.py index c85fd9950..00be4f282 100644 --- a/detection_rules/ml.py +++ b/detection_rules/ml.py @@ -383,7 +383,7 @@ def setup_bundle(ctx, model_tag, repo, model_dir): click.echo(table) click.echo('Associated rules and jobs can be found under ML-experimental-detections releases in the repo') - click.echo('To upload rules, run: kibana upload-rule ') + click.echo('To upload rules, run: kibana import-rules -f ') click.echo('To upload ML jobs, run: es experimental upload-ml-job ') diff --git a/docs-dev/custom-rules.md b/docs-dev/custom-rules-management.md similarity index 100% rename from docs-dev/custom-rules.md rename to docs-dev/custom-rules-management.md diff --git a/docs-dev/detections-as-code.md b/docs-dev/detections-as-code.md new file mode 100644 index 000000000..59611dcf5 --- /dev/null +++ b/docs-dev/detections-as-code.md @@ -0,0 +1,57 @@ +# Detection as Code (DaC) Components in Detection-Rules Repo + +The **detection-rules** repository contains features for **Detections as Code (DaC)**. These components, including CLI options and workflows, provide methods to help apply DaC principles in practice. The specific DaC architecture should be carefully considered before implementation, for more conceptual ideas and details about DaC, refer to the [DaC Documentation](https://dac-reference.readthedocs.io/en/latest/). Reference implementation is shared to facilitate experimentation and community contributions. + +> [!NOTE] +> This guidance outlines the support scope and best practices for using DaC components within the detection-rules repo. Users should take full responsibility for their usage of this repo, thoroughly test these tools in their environments, and verify functionality before using them. + +--- + +## Support and Scope + +Supported DaC components that interact with the Elastic Security Solution: +- kibana export-rules ([link](https://github.com/elastic/detection-rules/blob/main/CLI.md#exporting-rules)) +- kibana import-rules ([link](https://github.com/elastic/detection-rules/blob/main/CLI.md#using-import-rules)) +- import-rules-to-repo ([link](https://github.com/elastic/detection-rules/blob/main/CLI.md#import-rules-to-repo)) +- export-rules-from-repo ([link](https://github.com/elastic/detection-rules/blob/main/CLI.md#uploading-rules-to-kibana)) + +We welcome general questions, feature requests, and bug reports through the following channels: +- **GitHub Issues**: For raising general questions, bugs, and feature requests related to the detection-rules repo. + [GitHub Issues](https://github.com/elastic/detection-rules/issues) +- **Community Slack**: For informal discussions or questions (note that message history is limited to 30 days). + [Elastic Security Community Slack](https://elasticstack.slack.com/archives/C06TE19EP09) + +Support tickets related to this DaC reference implementation can be opened with Elastic, however since the logic is just a wrapper of the underlying product API's, we ask to resolve urgent DaC issues via direct interaction with the underlying [Kibana APIs](https://www.elastic.co/docs/api/doc/kibana/v8/group/endpoint-security-detections-api) or [Elastic Security Solution UI](https://www.elastic.co/guide/en/security/current/rules-ui-management.html), as we will not be able to treat DaC related support requests as a high severity (immediate time frame). + +> [!TIP] +> Questions about Kibana API usage should be directed to the Kibana repository: +> [Kibana Issues](https://github.com/elastic/kibana/issues) + +--- + +## Feature Requests + +Feature requests for the DaC components that interact with the Elastic Security Solution (`kibana export-rules`, `kibana import-rules`, `import-rules-to-repo`, and `export-rules-from-repo`) in this repository will be handled similarly to the rest of the detection-rules repo: +- **Prioritization**: Feature requests will be prioritized along with other development work in the repository. +- **Schema Updates**: If there are breaking schema changes in Kibana that affect importing/exporting detection rules, those changes will be prioritized. +- **Rule Engine API**: Current CLI tools leverage the rules engine API, and improvements to this will be treated as part of the ongoing development. +--- + +## Reference Implementation of DaC Components + +DaC is not a single tool. Detection as Code (DaC) is a modern security approach that applies software development best practices to the creation, management, and deployment of security rules. Here is a short summary of several components that extend upon Elastic's rule management capabilities (e.g. query validation, schema validation, unit tests, etc.) provided to help fast track users ability to implement custom DaC implementations in their private environments. If you are new to these concepts, please refer to the [DaC Documentation](https://dac-reference.readthedocs.io/en/latest/), which also provides a quickstart guide and example end-to-end CI/CD workflows. These components are configurable by using the [custom-rules](docs-dev/custom-rules.md) setup. + +- Kibana's Rule Versioning Mechanism ([link](https://dac-reference.readthedocs.io/en/latest/internals_of_the_detection_rules_repo.html#option-2-defer-to-elastic-security)) +- Local rule management (e.g. autoschema generation, actions and exceptions) ([link](https://dac-reference.readthedocs.io/en/latest/internals_of_the_detection_rules_repo.html#option-1-using-the-built-in-configuration)) + +--- + +## Best Practices for Using DaC Components + +When implementing DaC in your production environment, follow these best practices: + +- **Design and Test Rigorously**: Since every DaC implementation will be user-specific, remember to diligently design, and thoroughly test the tools before deploying them in a production environment. +- **Version Compatibility**: Before upgrading the detection-rules repo version, ensure that you test compatibility with your environment. For more information, see our [Versioning Documentation](https://github.com/elastic/ia-trade-team/issues/471%23issuecomment-2423259800). +- **Limited Backward Compatibility**: We do not guarantee backward compatibility across versions for rule schemas. While we aim to make new fields optional where feasible, there may be minimum version requirements for Elastic Stack and are subject to Kibana's rule schema definitions. +- **Schema Parity**: Not all fields in the schema defined in Kibana are fully supported. Some fields in the detection-rules repo are generalized (e.g., `field = dict()`), while others are more strictly defined. This is due to the complexity of the schemas and the prioritization of Elastic's internal use cases. + diff --git a/docs-dev/experimental-machine-learning/experimental-detections.md b/docs-dev/experimental-machine-learning/experimental-detections.md index 60d615fe3..2dbc3e585 100644 --- a/docs-dev/experimental-machine-learning/experimental-detections.md +++ b/docs-dev/experimental-machine-learning/experimental-detections.md @@ -14,7 +14,7 @@ Rules are now stored in ndjson format and can be imported into Kibana via the se Earlier releases stored the rules in toml format. These can be uploaded using the [7.12 branch](https://github.com/elastic/detection-rules/tree/7.12) CLI using the -[kibana upload-rule](../../CLI.md#uploading-rules-to-kibana) command +[kibana import-rules](../../CLI.md#uploading-rules-to-kibana) command ### Uploading ML Jobs and Datafeeds diff --git a/pyproject.toml b/pyproject.toml index 261b65766..d701b0cec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "detection_rules" -version = "0.4.18" +version = "0.4.19" description = "Detection Rules is the home for rules used by Elastic Security. This repository is used for the development, maintenance, testing, validation, and release of rules for Elastic Security’s Detection Engine." readme = "README.md" requires-python = ">=3.12"