diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 142cedc6f..01b738685 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,13 +1,13 @@ # detection-rules code owners # POC: Elastic Security Intelligence and Analytics Team -tests/**/*.py @brokensound77 @rw-access -detection_rules/ @brokensound77 @rw-access -tests/ @brokensound77 @rw-access +tests/**/*.py @brokensound77 @mikaayenson @terrancedejesus +detection_rules/ @brokensound77 @mikaayenson @terrancedejesus +tests/ @brokensound77 @mikaayenson @terrancedejesus # skip rta-mapping to avoid the spam -etc/packages.yml @brokensound77 @rw-access -etc/*.json @brokensound77 @rw-access -etc/*.json @brokensound77 @rw-access -etc/*/* @brokensound77 @rw-access +detection_rules/etc/packages.yml @brokensound77 @mikaayenson @terrancedejesus +detection_rules/etc/*.json @brokensound77 @mikaayenson @terrancedejesus +detection_rules/etc/*.json @brokensound77 @mikaayenson @terrancedejesus +detection_rules/etc/*/* @brokensound77 @mikaayenson @terrancedejesus diff --git a/.github/paths-labeller.yml b/.github/paths-labeller.yml index 024c69908..1205b6bd4 100644 --- a/.github/paths-labeller.yml +++ b/.github/paths-labeller.yml @@ -3,10 +3,10 @@ - "./**/*.md" - "schema": - "detection_rules/beats.py" - - "etc/beats_schemas/**/*" + - "detection_rules/etc/beats_schemas/**/*" - "detection_rules/ecs.py" - - "etc/ecs_schemas/**/*" - - "etc/api_schemas/**/*" + - "detection_rules/etc/ecs_schemas/**/*" + - "detection_rules/etc/api_schemas/**/*" - "detection_rules/schemas/**/*" - "python": - "detection_rules/**/*.py" diff --git a/.github/workflows/lock-versions.yml b/.github/workflows/lock-versions.yml index 417b65cce..45b27b7f5 100644 --- a/.github/workflows/lock-versions.yml +++ b/.github/workflows/lock-versions.yml @@ -49,8 +49,8 @@ jobs: env: BRANCHES: "${{github.event.inputs.branches}}" run: | - ./etc/lock-multiple.sh $BRANCHES - git add etc/version.lock.json + ./detection_rules/etc/lock-multiple.sh $BRANCHES + git add detection_rules/etc/version.lock.json - name: Create Pull Request uses: peter-evans/create-pull-request@v3 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0ed9b897a..dce8244b7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -8,26 +8,28 @@ These guidelines will also help you post meaningful issues that will be more eas ## Table of Contents -- [Effective issue creation in Detection Rules](#effective-issue-creation-in-detection-rules) - - [Why we create issues before contributing code or new rules](#why-we-create-issues-before-contributing-code-or-new-rules) - - [What a good issue looks like](#what-a-good-issue-looks-like) - - ["My issue isn’t getting enough attention"](#my-issue-isnt-getting-enough-attention) - - ["I want to help!"](#i-want-to-help) -- [How we use Git and GitHub](#how-we-use-git-and-github) - - [Forking](#forking) - - [Branching](#branching) - - [Commit messages](#commit-messages) - - [What goes into a Pull Request](#what-goes-into-a-pull-request) -- [Our approach to detection engineering](#our-approach-to-detection-engineering) - - [Rule metadata](#rule-metadata) - - [Using Elastic Common Schema (ECS)](#using-elastic-common-schema-ecs) - - [Creating a rule with the CLI](#creating-a-rule-with-the-cli) - - [Testing a rule with the CLI](#testing-a-rule-with-the-cli) -- [Writing style](#writing-style) -- [Signing the contributor license agreement](#signing-the-contributor-license-agreement) -- [Submitting a Pull Request](#submitting-a-pull-request) - - [What to expect from a code review](#what-to-expect-from-a-code-review) - - [How we handle merges](#how-we-handle-merges) +- [Contributing to Detection Rules](#contributing-to-detection-rules) + - [Table of Contents](#table-of-contents) + - [Effective issue creation in Detection Rules](#effective-issue-creation-in-detection-rules) + - [Why we create issues before contributing code or new rules](#why-we-create-issues-before-contributing-code-or-new-rules) + - [What a good issue looks like](#what-a-good-issue-looks-like) + - ["My issue isn't getting enough attention"](#my-issue-isnt-getting-enough-attention) + - ["I want to help!"](#i-want-to-help) + - [How we use Git and GitHub](#how-we-use-git-and-github) + - [Forking](#forking) + - [Branching](#branching) + - [Commit messages](#commit-messages) + - [What goes into a Pull Request](#what-goes-into-a-pull-request) + - [Our approach to detection engineering](#our-approach-to-detection-engineering) + - [Rule metadata](#rule-metadata) + - [Using Elastic Common Schema (ECS)](#using-elastic-common-schema-ecs) + - [Creating a rule with the CLI](#creating-a-rule-with-the-cli) + - [Testing a rule with the CLI](#testing-a-rule-with-the-cli) + - [Writing style](#writing-style) + - [Signing the contributor license agreement](#signing-the-contributor-license-agreement) + - [Submitting a Pull Request](#submitting-a-pull-request) + - [What to expect from a code review](#what-to-expect-from-a-code-review) + - [How we handle merges](#how-we-handle-merges) ## Effective issue creation in Detection Rules @@ -167,7 +169,7 @@ Our rules should be written generically when possible. We use [Elastic Common Sc If the relevant [categorization values](https://www.elastic.co/guide/en/ecs/current/ecs-category-field-values-reference.html) are already defined for ECS, we use these to narrow down the event type before adding the query. Typically, the query starts with the broadest grouping possible and gets narrower for each clause. For example, we might write `event.category:process and event.type:start and process.name:net.exe and process.args:group`. First, we match process events with `event.category`, then narrow to creation events with `event.type`. Of the process creation events, we're looking for the process `net.exe` with `process.name` and finally we check the arguments `group` by looking at `process.args`. This flow has little effect on the generated Elasticsearch query, but is the most intuitive to read for rule developers. -Sometimes, it might not make sense for ECS to standardize a field, value, or category. Occasionally, we may encounter fields that specific to a single use-case or vendor. When that happens, we add an exception in [etc/non-ecs-schema.json](etc/non-ecs-schema.json). We automatically detect beats by looking at the index patterns used in a rule. If we see `winlogbeat-*`, for example, then we can validate the rule against ECS + Winlogbeat. When using a particular beat, please use `event.module` and `event.dataset` to make the rule more precise and to better nudge the validation logic. Similar to our logic flow for ECS categorization, we recommend searches progress from `event.module` → `event.dataset` → `event.action` → ``. +Sometimes, it might not make sense for ECS to standardize a field, value, or category. Occasionally, we may encounter fields that specific to a single use-case or vendor. When that happens, we add an exception in [detection_rules/etc/non-ecs-schema.json](detection_rules/etc/non-ecs-schema.json). We automatically detect beats by looking at the index patterns used in a rule. If we see `winlogbeat-*`, for example, then we can validate the rule against ECS + Winlogbeat. When using a particular beat, please use `event.module` and `event.dataset` to make the rule more precise and to better nudge the validation logic. Similar to our logic flow for ECS categorization, we recommend searches progress from `event.module` → `event.dataset` → `event.action` → ``. When a Pull Request is missing a necessary ECS change, please add an issue to [elastic/ecs](https://github.com/elastic/ecs) and link it from the pull request. We don't want to leave PRs blocked for too long, so if the ECS issue isn't progressing, then we can add a note and use the vendor- or beat-specific fields. We'll create another issue, reminding us to update the rule logic to switch to the ECS field when it becomes available. To maximize compatibility, we may add an `or` clause for a release or two to handle the different permutatations. After a few releases, we'll remove this and strictly require the ECS fields. @@ -202,7 +204,7 @@ references (multi, comma separated): risk_score [21] ("n/a" to leave blank) (required): rule_id [90d0c543-e197-46d8-934d-0320b2c83486] ("n/a" to leave blank) (required): severity [low] ("n/a" to leave blank) (required): medium -tags (multi, comma separated): Windows +tags (multi, comma separated): Windows throttle: timeline_id: timeline_title: diff --git a/README.md b/README.md index a5a8e1068..835f50e18 100644 --- a/README.md +++ b/README.md @@ -11,12 +11,13 @@ This repository was first announced on Elastic's blog post, [Elastic Security op ## Table of Contents -- [Overview of this repository](#overview-of-this-repository) -- [Getting started](#getting-started) -- [Red Team Automation](rta) -- [How to contribute](#how-to-contribute) -- [Licensing](#licensing) -- [Questions? Problems? Suggestions?](#questions-problems-suggestions) +- [Detection Rules](#detection-rules) + - [Table of Contents](#table-of-contents) + - [Overview of this repository](#overview-of-this-repository) + - [Getting started](#getting-started) + - [How to contribute](#how-to-contribute) + - [Licensing](#licensing) + - [Questions? Problems? Suggestions?](#questions-problems-suggestions) ## Overview of this repository @@ -26,7 +27,7 @@ Detection Rules contains more than just static rule files. This repository also | folder | description | |-------------------------------------- |------------------------------------------------------------------------------------ | | [`detection_rules/`](detection_rules) | Python module for rule parsing, validating and packaging | -| [`etc/`](etc) | Miscellaneous files, such as ECS and Beats schemas | +| [`detection_rules/etc/`](etc) | Miscellaneous files, such as ECS and Beats schemas | | [`kibana/`](kibana) | Python library for handling the API calls to Kibana and the Detection Engine | | [`kql/`](kql) | Python library for parsing and validating Kibana Query Language | | [`rta/`](rta) | Red Team Automation code used to emulate attacker techniques, used for rule testing | @@ -43,13 +44,13 @@ Collecting jsl==0.2.4 Downloading jsl-0.2.4.tar.gz (21 kB) Collecting jsonschema==3.2.0 Downloading jsonschema-3.2.0-py2.py3-none-any.whl (56 kB) - |████████████████████████████████| 56 kB 318 kB/s + |████████████████████████████████| 56 kB 318 kB/s Collecting requests==2.22.0 Downloading requests-2.22.0-py2.py3-none-any.whl (57 kB) - |████████████████████████████████| 57 kB 1.2 MB/s + |████████████████████████████████| 57 kB 1.2 MB/s Collecting Click==7.0 Downloading Click-7.0-py2.py3-none-any.whl (81 kB) - |████████████████████████████████| 81 kB 2.6 MB/s + |████████████████████████████████| 81 kB 2.6 MB/s ... ``` diff --git a/detection_rules/beats.py b/detection_rules/beats.py index fe3308e3f..972c6da81 100644 --- a/detection_rules/beats.py +++ b/detection_rules/beats.py @@ -56,7 +56,7 @@ def _decompress_and_save_schema(url, release_name): # remove all non-beat directories fs = {k: v for k, v in fs.get("folders", {}).items() if k.endswith("beat")} - print(f"Saving etc/beats_schema/{release_name}.json") + print(f"Saving detection_rules/etc/beats_schema/{release_name}.json") compressed = gzip_compress(json.dumps(fs, sort_keys=True, cls=DateTimeEncoder)) path = get_etc_path("beats_schemas", release_name + ".json.gz") diff --git a/detection_rules/devtools.py b/detection_rules/devtools.py index 1b59de755..511a4f7a4 100644 --- a/detection_rules/devtools.py +++ b/detection_rules/devtools.py @@ -189,7 +189,7 @@ class GitChangeEntry: def prune_staging_area(target_stack_version: str, dry_run: bool): """Prune the git staging area to remove changes to incompatible rules.""" exceptions = { - "etc/packages.yml", + "detection_rules/etc/packages.yml", } target_stack_version = Version(target_stack_version)[:2] diff --git a/etc/api_schemas/7.10/7.10.base.json b/detection_rules/etc/api_schemas/7.10/7.10.base.json similarity index 100% rename from etc/api_schemas/7.10/7.10.base.json rename to detection_rules/etc/api_schemas/7.10/7.10.base.json diff --git a/etc/api_schemas/7.10/7.10.eql.json b/detection_rules/etc/api_schemas/7.10/7.10.eql.json similarity index 100% rename from etc/api_schemas/7.10/7.10.eql.json rename to detection_rules/etc/api_schemas/7.10/7.10.eql.json diff --git a/etc/api_schemas/7.10/7.10.machine_learning.json b/detection_rules/etc/api_schemas/7.10/7.10.machine_learning.json similarity index 100% rename from etc/api_schemas/7.10/7.10.machine_learning.json rename to detection_rules/etc/api_schemas/7.10/7.10.machine_learning.json diff --git a/etc/api_schemas/7.10/7.10.query.json b/detection_rules/etc/api_schemas/7.10/7.10.query.json similarity index 100% rename from etc/api_schemas/7.10/7.10.query.json rename to detection_rules/etc/api_schemas/7.10/7.10.query.json diff --git a/etc/api_schemas/7.10/7.10.saved_query.json b/detection_rules/etc/api_schemas/7.10/7.10.saved_query.json similarity index 100% rename from etc/api_schemas/7.10/7.10.saved_query.json rename to detection_rules/etc/api_schemas/7.10/7.10.saved_query.json diff --git a/etc/api_schemas/7.10/7.10.threshold.json b/detection_rules/etc/api_schemas/7.10/7.10.threshold.json similarity index 100% rename from etc/api_schemas/7.10/7.10.threshold.json rename to detection_rules/etc/api_schemas/7.10/7.10.threshold.json diff --git a/etc/api_schemas/7.11/7.11.base.json b/detection_rules/etc/api_schemas/7.11/7.11.base.json similarity index 100% rename from etc/api_schemas/7.11/7.11.base.json rename to detection_rules/etc/api_schemas/7.11/7.11.base.json diff --git a/etc/api_schemas/7.11/7.11.eql.json b/detection_rules/etc/api_schemas/7.11/7.11.eql.json similarity index 100% rename from etc/api_schemas/7.11/7.11.eql.json rename to detection_rules/etc/api_schemas/7.11/7.11.eql.json diff --git a/etc/api_schemas/7.11/7.11.machine_learning.json b/detection_rules/etc/api_schemas/7.11/7.11.machine_learning.json similarity index 100% rename from etc/api_schemas/7.11/7.11.machine_learning.json rename to detection_rules/etc/api_schemas/7.11/7.11.machine_learning.json diff --git a/etc/api_schemas/7.11/7.11.query.json b/detection_rules/etc/api_schemas/7.11/7.11.query.json similarity index 100% rename from etc/api_schemas/7.11/7.11.query.json rename to detection_rules/etc/api_schemas/7.11/7.11.query.json diff --git a/etc/api_schemas/7.11/7.11.saved_query.json b/detection_rules/etc/api_schemas/7.11/7.11.saved_query.json similarity index 100% rename from etc/api_schemas/7.11/7.11.saved_query.json rename to detection_rules/etc/api_schemas/7.11/7.11.saved_query.json diff --git a/etc/api_schemas/7.11/7.11.threshold.json b/detection_rules/etc/api_schemas/7.11/7.11.threshold.json similarity index 100% rename from etc/api_schemas/7.11/7.11.threshold.json rename to detection_rules/etc/api_schemas/7.11/7.11.threshold.json diff --git a/etc/api_schemas/7.12/7.12.base.json b/detection_rules/etc/api_schemas/7.12/7.12.base.json similarity index 100% rename from etc/api_schemas/7.12/7.12.base.json rename to detection_rules/etc/api_schemas/7.12/7.12.base.json diff --git a/etc/api_schemas/7.12/7.12.eql.json b/detection_rules/etc/api_schemas/7.12/7.12.eql.json similarity index 100% rename from etc/api_schemas/7.12/7.12.eql.json rename to detection_rules/etc/api_schemas/7.12/7.12.eql.json diff --git a/etc/api_schemas/7.12/7.12.machine_learning.json b/detection_rules/etc/api_schemas/7.12/7.12.machine_learning.json similarity index 100% rename from etc/api_schemas/7.12/7.12.machine_learning.json rename to detection_rules/etc/api_schemas/7.12/7.12.machine_learning.json diff --git a/etc/api_schemas/7.12/7.12.query.json b/detection_rules/etc/api_schemas/7.12/7.12.query.json similarity index 100% rename from etc/api_schemas/7.12/7.12.query.json rename to detection_rules/etc/api_schemas/7.12/7.12.query.json diff --git a/etc/api_schemas/7.12/7.12.saved_query.json b/detection_rules/etc/api_schemas/7.12/7.12.saved_query.json similarity index 100% rename from etc/api_schemas/7.12/7.12.saved_query.json rename to detection_rules/etc/api_schemas/7.12/7.12.saved_query.json diff --git a/etc/api_schemas/7.12/7.12.threshold.json b/detection_rules/etc/api_schemas/7.12/7.12.threshold.json similarity index 100% rename from etc/api_schemas/7.12/7.12.threshold.json rename to detection_rules/etc/api_schemas/7.12/7.12.threshold.json diff --git a/etc/api_schemas/7.13/7.13.base.json b/detection_rules/etc/api_schemas/7.13/7.13.base.json similarity index 100% rename from etc/api_schemas/7.13/7.13.base.json rename to detection_rules/etc/api_schemas/7.13/7.13.base.json diff --git a/etc/api_schemas/7.13/7.13.eql.json b/detection_rules/etc/api_schemas/7.13/7.13.eql.json similarity index 100% rename from etc/api_schemas/7.13/7.13.eql.json rename to detection_rules/etc/api_schemas/7.13/7.13.eql.json diff --git a/etc/api_schemas/7.13/7.13.machine_learning.json b/detection_rules/etc/api_schemas/7.13/7.13.machine_learning.json similarity index 100% rename from etc/api_schemas/7.13/7.13.machine_learning.json rename to detection_rules/etc/api_schemas/7.13/7.13.machine_learning.json diff --git a/etc/api_schemas/7.13/7.13.query.json b/detection_rules/etc/api_schemas/7.13/7.13.query.json similarity index 100% rename from etc/api_schemas/7.13/7.13.query.json rename to detection_rules/etc/api_schemas/7.13/7.13.query.json diff --git a/etc/api_schemas/7.13/7.13.threat_match.json b/detection_rules/etc/api_schemas/7.13/7.13.threat_match.json similarity index 100% rename from etc/api_schemas/7.13/7.13.threat_match.json rename to detection_rules/etc/api_schemas/7.13/7.13.threat_match.json diff --git a/etc/api_schemas/7.13/7.13.threshold.json b/detection_rules/etc/api_schemas/7.13/7.13.threshold.json similarity index 100% rename from etc/api_schemas/7.13/7.13.threshold.json rename to detection_rules/etc/api_schemas/7.13/7.13.threshold.json diff --git a/etc/api_schemas/7.14/7.14.base.json b/detection_rules/etc/api_schemas/7.14/7.14.base.json similarity index 100% rename from etc/api_schemas/7.14/7.14.base.json rename to detection_rules/etc/api_schemas/7.14/7.14.base.json diff --git a/etc/api_schemas/7.14/7.14.eql.json b/detection_rules/etc/api_schemas/7.14/7.14.eql.json similarity index 100% rename from etc/api_schemas/7.14/7.14.eql.json rename to detection_rules/etc/api_schemas/7.14/7.14.eql.json diff --git a/etc/api_schemas/7.14/7.14.machine_learning.json b/detection_rules/etc/api_schemas/7.14/7.14.machine_learning.json similarity index 100% rename from etc/api_schemas/7.14/7.14.machine_learning.json rename to detection_rules/etc/api_schemas/7.14/7.14.machine_learning.json diff --git a/etc/api_schemas/7.14/7.14.query.json b/detection_rules/etc/api_schemas/7.14/7.14.query.json similarity index 100% rename from etc/api_schemas/7.14/7.14.query.json rename to detection_rules/etc/api_schemas/7.14/7.14.query.json diff --git a/etc/api_schemas/7.14/7.14.threat_match.json b/detection_rules/etc/api_schemas/7.14/7.14.threat_match.json similarity index 100% rename from etc/api_schemas/7.14/7.14.threat_match.json rename to detection_rules/etc/api_schemas/7.14/7.14.threat_match.json diff --git a/etc/api_schemas/7.14/7.14.threshold.json b/detection_rules/etc/api_schemas/7.14/7.14.threshold.json similarity index 100% rename from etc/api_schemas/7.14/7.14.threshold.json rename to detection_rules/etc/api_schemas/7.14/7.14.threshold.json diff --git a/etc/api_schemas/7.15/7.15.base.json b/detection_rules/etc/api_schemas/7.15/7.15.base.json similarity index 100% rename from etc/api_schemas/7.15/7.15.base.json rename to detection_rules/etc/api_schemas/7.15/7.15.base.json diff --git a/etc/api_schemas/7.15/7.15.eql.json b/detection_rules/etc/api_schemas/7.15/7.15.eql.json similarity index 100% rename from etc/api_schemas/7.15/7.15.eql.json rename to detection_rules/etc/api_schemas/7.15/7.15.eql.json diff --git a/etc/api_schemas/7.15/7.15.machine_learning.json b/detection_rules/etc/api_schemas/7.15/7.15.machine_learning.json similarity index 100% rename from etc/api_schemas/7.15/7.15.machine_learning.json rename to detection_rules/etc/api_schemas/7.15/7.15.machine_learning.json diff --git a/etc/api_schemas/7.15/7.15.query.json b/detection_rules/etc/api_schemas/7.15/7.15.query.json similarity index 100% rename from etc/api_schemas/7.15/7.15.query.json rename to detection_rules/etc/api_schemas/7.15/7.15.query.json diff --git a/etc/api_schemas/7.15/7.15.threat_match.json b/detection_rules/etc/api_schemas/7.15/7.15.threat_match.json similarity index 100% rename from etc/api_schemas/7.15/7.15.threat_match.json rename to detection_rules/etc/api_schemas/7.15/7.15.threat_match.json diff --git a/etc/api_schemas/7.15/7.15.threshold.json b/detection_rules/etc/api_schemas/7.15/7.15.threshold.json similarity index 100% rename from etc/api_schemas/7.15/7.15.threshold.json rename to detection_rules/etc/api_schemas/7.15/7.15.threshold.json diff --git a/etc/api_schemas/7.16/7.16.base.json b/detection_rules/etc/api_schemas/7.16/7.16.base.json similarity index 100% rename from etc/api_schemas/7.16/7.16.base.json rename to detection_rules/etc/api_schemas/7.16/7.16.base.json diff --git a/etc/api_schemas/7.16/7.16.eql.json b/detection_rules/etc/api_schemas/7.16/7.16.eql.json similarity index 100% rename from etc/api_schemas/7.16/7.16.eql.json rename to detection_rules/etc/api_schemas/7.16/7.16.eql.json diff --git a/etc/api_schemas/7.16/7.16.machine_learning.json b/detection_rules/etc/api_schemas/7.16/7.16.machine_learning.json similarity index 100% rename from etc/api_schemas/7.16/7.16.machine_learning.json rename to detection_rules/etc/api_schemas/7.16/7.16.machine_learning.json diff --git a/etc/api_schemas/7.16/7.16.query.json b/detection_rules/etc/api_schemas/7.16/7.16.query.json similarity index 100% rename from etc/api_schemas/7.16/7.16.query.json rename to detection_rules/etc/api_schemas/7.16/7.16.query.json diff --git a/etc/api_schemas/7.16/7.16.threat_match.json b/detection_rules/etc/api_schemas/7.16/7.16.threat_match.json similarity index 100% rename from etc/api_schemas/7.16/7.16.threat_match.json rename to detection_rules/etc/api_schemas/7.16/7.16.threat_match.json diff --git a/etc/api_schemas/7.16/7.16.threshold.json b/detection_rules/etc/api_schemas/7.16/7.16.threshold.json similarity index 100% rename from etc/api_schemas/7.16/7.16.threshold.json rename to detection_rules/etc/api_schemas/7.16/7.16.threshold.json diff --git a/etc/api_schemas/7.8/7.8.base.json b/detection_rules/etc/api_schemas/7.8/7.8.base.json similarity index 100% rename from etc/api_schemas/7.8/7.8.base.json rename to detection_rules/etc/api_schemas/7.8/7.8.base.json diff --git a/etc/api_schemas/7.8/7.8.machine_learning.json b/detection_rules/etc/api_schemas/7.8/7.8.machine_learning.json similarity index 100% rename from etc/api_schemas/7.8/7.8.machine_learning.json rename to detection_rules/etc/api_schemas/7.8/7.8.machine_learning.json diff --git a/etc/api_schemas/7.8/7.8.query.json b/detection_rules/etc/api_schemas/7.8/7.8.query.json similarity index 100% rename from etc/api_schemas/7.8/7.8.query.json rename to detection_rules/etc/api_schemas/7.8/7.8.query.json diff --git a/etc/api_schemas/7.8/7.8.saved_query.json b/detection_rules/etc/api_schemas/7.8/7.8.saved_query.json similarity index 100% rename from etc/api_schemas/7.8/7.8.saved_query.json rename to detection_rules/etc/api_schemas/7.8/7.8.saved_query.json diff --git a/etc/api_schemas/7.9/7.9.base.json b/detection_rules/etc/api_schemas/7.9/7.9.base.json similarity index 100% rename from etc/api_schemas/7.9/7.9.base.json rename to detection_rules/etc/api_schemas/7.9/7.9.base.json diff --git a/etc/api_schemas/7.9/7.9.machine_learning.json b/detection_rules/etc/api_schemas/7.9/7.9.machine_learning.json similarity index 100% rename from etc/api_schemas/7.9/7.9.machine_learning.json rename to detection_rules/etc/api_schemas/7.9/7.9.machine_learning.json diff --git a/etc/api_schemas/7.9/7.9.query.json b/detection_rules/etc/api_schemas/7.9/7.9.query.json similarity index 100% rename from etc/api_schemas/7.9/7.9.query.json rename to detection_rules/etc/api_schemas/7.9/7.9.query.json diff --git a/etc/api_schemas/7.9/7.9.saved_query.json b/detection_rules/etc/api_schemas/7.9/7.9.saved_query.json similarity index 100% rename from etc/api_schemas/7.9/7.9.saved_query.json rename to detection_rules/etc/api_schemas/7.9/7.9.saved_query.json diff --git a/etc/api_schemas/7.9/7.9.threshold.json b/detection_rules/etc/api_schemas/7.9/7.9.threshold.json similarity index 100% rename from etc/api_schemas/7.9/7.9.threshold.json rename to detection_rules/etc/api_schemas/7.9/7.9.threshold.json diff --git a/etc/api_schemas/8.0/8.0.base.json b/detection_rules/etc/api_schemas/8.0/8.0.base.json similarity index 100% rename from etc/api_schemas/8.0/8.0.base.json rename to detection_rules/etc/api_schemas/8.0/8.0.base.json diff --git a/etc/api_schemas/8.0/8.0.eql.json b/detection_rules/etc/api_schemas/8.0/8.0.eql.json similarity index 100% rename from etc/api_schemas/8.0/8.0.eql.json rename to detection_rules/etc/api_schemas/8.0/8.0.eql.json diff --git a/etc/api_schemas/8.0/8.0.machine_learning.json b/detection_rules/etc/api_schemas/8.0/8.0.machine_learning.json similarity index 100% rename from etc/api_schemas/8.0/8.0.machine_learning.json rename to detection_rules/etc/api_schemas/8.0/8.0.machine_learning.json diff --git a/etc/api_schemas/8.0/8.0.query.json b/detection_rules/etc/api_schemas/8.0/8.0.query.json similarity index 100% rename from etc/api_schemas/8.0/8.0.query.json rename to detection_rules/etc/api_schemas/8.0/8.0.query.json diff --git a/etc/api_schemas/8.0/8.0.threat_match.json b/detection_rules/etc/api_schemas/8.0/8.0.threat_match.json similarity index 100% rename from etc/api_schemas/8.0/8.0.threat_match.json rename to detection_rules/etc/api_schemas/8.0/8.0.threat_match.json diff --git a/etc/api_schemas/8.0/8.0.threshold.json b/detection_rules/etc/api_schemas/8.0/8.0.threshold.json similarity index 100% rename from etc/api_schemas/8.0/8.0.threshold.json rename to detection_rules/etc/api_schemas/8.0/8.0.threshold.json diff --git a/etc/api_schemas/8.1/8.1.base.json b/detection_rules/etc/api_schemas/8.1/8.1.base.json similarity index 100% rename from etc/api_schemas/8.1/8.1.base.json rename to detection_rules/etc/api_schemas/8.1/8.1.base.json diff --git a/etc/api_schemas/8.1/8.1.eql.json b/detection_rules/etc/api_schemas/8.1/8.1.eql.json similarity index 100% rename from etc/api_schemas/8.1/8.1.eql.json rename to detection_rules/etc/api_schemas/8.1/8.1.eql.json diff --git a/etc/api_schemas/8.1/8.1.machine_learning.json b/detection_rules/etc/api_schemas/8.1/8.1.machine_learning.json similarity index 100% rename from etc/api_schemas/8.1/8.1.machine_learning.json rename to detection_rules/etc/api_schemas/8.1/8.1.machine_learning.json diff --git a/etc/api_schemas/8.1/8.1.query.json b/detection_rules/etc/api_schemas/8.1/8.1.query.json similarity index 100% rename from etc/api_schemas/8.1/8.1.query.json rename to detection_rules/etc/api_schemas/8.1/8.1.query.json diff --git a/etc/api_schemas/8.1/8.1.threat_match.json b/detection_rules/etc/api_schemas/8.1/8.1.threat_match.json similarity index 100% rename from etc/api_schemas/8.1/8.1.threat_match.json rename to detection_rules/etc/api_schemas/8.1/8.1.threat_match.json diff --git a/etc/api_schemas/8.1/8.1.threshold.json b/detection_rules/etc/api_schemas/8.1/8.1.threshold.json similarity index 100% rename from etc/api_schemas/8.1/8.1.threshold.json rename to detection_rules/etc/api_schemas/8.1/8.1.threshold.json diff --git a/etc/api_schemas/8.2/8.2.base.json b/detection_rules/etc/api_schemas/8.2/8.2.base.json similarity index 100% rename from etc/api_schemas/8.2/8.2.base.json rename to detection_rules/etc/api_schemas/8.2/8.2.base.json diff --git a/etc/api_schemas/8.2/8.2.eql.json b/detection_rules/etc/api_schemas/8.2/8.2.eql.json similarity index 100% rename from etc/api_schemas/8.2/8.2.eql.json rename to detection_rules/etc/api_schemas/8.2/8.2.eql.json diff --git a/etc/api_schemas/8.2/8.2.machine_learning.json b/detection_rules/etc/api_schemas/8.2/8.2.machine_learning.json similarity index 100% rename from etc/api_schemas/8.2/8.2.machine_learning.json rename to detection_rules/etc/api_schemas/8.2/8.2.machine_learning.json diff --git a/etc/api_schemas/8.2/8.2.query.json b/detection_rules/etc/api_schemas/8.2/8.2.query.json similarity index 100% rename from etc/api_schemas/8.2/8.2.query.json rename to detection_rules/etc/api_schemas/8.2/8.2.query.json diff --git a/etc/api_schemas/8.2/8.2.threat_match.json b/detection_rules/etc/api_schemas/8.2/8.2.threat_match.json similarity index 100% rename from etc/api_schemas/8.2/8.2.threat_match.json rename to detection_rules/etc/api_schemas/8.2/8.2.threat_match.json diff --git a/etc/api_schemas/8.2/8.2.threshold.json b/detection_rules/etc/api_schemas/8.2/8.2.threshold.json similarity index 100% rename from etc/api_schemas/8.2/8.2.threshold.json rename to detection_rules/etc/api_schemas/8.2/8.2.threshold.json diff --git a/etc/api_schemas/master/master.base.json b/detection_rules/etc/api_schemas/master/master.base.json similarity index 100% rename from etc/api_schemas/master/master.base.json rename to detection_rules/etc/api_schemas/master/master.base.json diff --git a/etc/api_schemas/master/master.eql.json b/detection_rules/etc/api_schemas/master/master.eql.json similarity index 100% rename from etc/api_schemas/master/master.eql.json rename to detection_rules/etc/api_schemas/master/master.eql.json diff --git a/etc/api_schemas/master/master.machine_learning.json b/detection_rules/etc/api_schemas/master/master.machine_learning.json similarity index 100% rename from etc/api_schemas/master/master.machine_learning.json rename to detection_rules/etc/api_schemas/master/master.machine_learning.json diff --git a/etc/api_schemas/master/master.query.json b/detection_rules/etc/api_schemas/master/master.query.json similarity index 100% rename from etc/api_schemas/master/master.query.json rename to detection_rules/etc/api_schemas/master/master.query.json diff --git a/etc/api_schemas/master/master.threat_match.json b/detection_rules/etc/api_schemas/master/master.threat_match.json similarity index 100% rename from etc/api_schemas/master/master.threat_match.json rename to detection_rules/etc/api_schemas/master/master.threat_match.json diff --git a/etc/api_schemas/master/master.threshold.json b/detection_rules/etc/api_schemas/master/master.threshold.json similarity index 100% rename from etc/api_schemas/master/master.threshold.json rename to detection_rules/etc/api_schemas/master/master.threshold.json diff --git a/etc/attack-crosswalk.json b/detection_rules/etc/attack-crosswalk.json similarity index 100% rename from etc/attack-crosswalk.json rename to detection_rules/etc/attack-crosswalk.json diff --git a/etc/attack-technique-redirects.json b/detection_rules/etc/attack-technique-redirects.json similarity index 100% rename from etc/attack-technique-redirects.json rename to detection_rules/etc/attack-technique-redirects.json diff --git a/etc/attack-v10.1.json.gz b/detection_rules/etc/attack-v10.1.json.gz similarity index 100% rename from etc/attack-v10.1.json.gz rename to detection_rules/etc/attack-v10.1.json.gz diff --git a/etc/beats_schemas/main.json.gz b/detection_rules/etc/beats_schemas/main.json.gz similarity index 100% rename from etc/beats_schemas/main.json.gz rename to detection_rules/etc/beats_schemas/main.json.gz diff --git a/etc/beats_schemas/v7.10.0.json.gz b/detection_rules/etc/beats_schemas/v7.10.0.json.gz similarity index 100% rename from etc/beats_schemas/v7.10.0.json.gz rename to detection_rules/etc/beats_schemas/v7.10.0.json.gz diff --git a/etc/beats_schemas/v7.11.2.json.gz b/detection_rules/etc/beats_schemas/v7.11.2.json.gz similarity index 100% rename from etc/beats_schemas/v7.11.2.json.gz rename to detection_rules/etc/beats_schemas/v7.11.2.json.gz diff --git a/etc/beats_schemas/v7.12.0.json.gz b/detection_rules/etc/beats_schemas/v7.12.0.json.gz similarity index 100% rename from etc/beats_schemas/v7.12.0.json.gz rename to detection_rules/etc/beats_schemas/v7.12.0.json.gz diff --git a/etc/beats_schemas/v7.13.2.json.gz b/detection_rules/etc/beats_schemas/v7.13.2.json.gz similarity index 100% rename from etc/beats_schemas/v7.13.2.json.gz rename to detection_rules/etc/beats_schemas/v7.13.2.json.gz diff --git a/etc/beats_schemas/v7.14.0.json.gz b/detection_rules/etc/beats_schemas/v7.14.0.json.gz similarity index 100% rename from etc/beats_schemas/v7.14.0.json.gz rename to detection_rules/etc/beats_schemas/v7.14.0.json.gz diff --git a/etc/beats_schemas/v7.15.1.json.gz b/detection_rules/etc/beats_schemas/v7.15.1.json.gz similarity index 100% rename from etc/beats_schemas/v7.15.1.json.gz rename to detection_rules/etc/beats_schemas/v7.15.1.json.gz diff --git a/etc/beats_schemas/v7.16.2.json.gz b/detection_rules/etc/beats_schemas/v7.16.2.json.gz similarity index 100% rename from etc/beats_schemas/v7.16.2.json.gz rename to detection_rules/etc/beats_schemas/v7.16.2.json.gz diff --git a/etc/beats_schemas/v7.7.0.json.gz b/detection_rules/etc/beats_schemas/v7.7.0.json.gz similarity index 100% rename from etc/beats_schemas/v7.7.0.json.gz rename to detection_rules/etc/beats_schemas/v7.7.0.json.gz diff --git a/etc/beats_schemas/v7.8.1.json.gz b/detection_rules/etc/beats_schemas/v7.8.1.json.gz similarity index 100% rename from etc/beats_schemas/v7.8.1.json.gz rename to detection_rules/etc/beats_schemas/v7.8.1.json.gz diff --git a/etc/beats_schemas/v7.9.2.json.gz b/detection_rules/etc/beats_schemas/v7.9.2.json.gz similarity index 100% rename from etc/beats_schemas/v7.9.2.json.gz rename to detection_rules/etc/beats_schemas/v7.9.2.json.gz diff --git a/etc/beats_schemas/v8.0.0-rc1.json.gz b/detection_rules/etc/beats_schemas/v8.0.0-rc1.json.gz similarity index 100% rename from etc/beats_schemas/v8.0.0-rc1.json.gz rename to detection_rules/etc/beats_schemas/v8.0.0-rc1.json.gz diff --git a/etc/deprecated_rules.json b/detection_rules/etc/deprecated_rules.json similarity index 100% rename from etc/deprecated_rules.json rename to detection_rules/etc/deprecated_rules.json diff --git a/etc/ecs_schemas/1.0.1/ecs_flat.json.gz b/detection_rules/etc/ecs_schemas/1.0.1/ecs_flat.json.gz similarity index 100% rename from etc/ecs_schemas/1.0.1/ecs_flat.json.gz rename to detection_rules/etc/ecs_schemas/1.0.1/ecs_flat.json.gz diff --git a/etc/ecs_schemas/1.0.1/ecs_nested.json.gz b/detection_rules/etc/ecs_schemas/1.0.1/ecs_nested.json.gz similarity index 100% rename from etc/ecs_schemas/1.0.1/ecs_nested.json.gz rename to detection_rules/etc/ecs_schemas/1.0.1/ecs_nested.json.gz diff --git a/etc/ecs_schemas/1.1.0/ecs_flat.json.gz b/detection_rules/etc/ecs_schemas/1.1.0/ecs_flat.json.gz similarity index 100% rename from etc/ecs_schemas/1.1.0/ecs_flat.json.gz rename to detection_rules/etc/ecs_schemas/1.1.0/ecs_flat.json.gz diff --git a/etc/ecs_schemas/1.1.0/ecs_nested.json.gz b/detection_rules/etc/ecs_schemas/1.1.0/ecs_nested.json.gz similarity index 100% rename from etc/ecs_schemas/1.1.0/ecs_nested.json.gz rename to detection_rules/etc/ecs_schemas/1.1.0/ecs_nested.json.gz diff --git a/etc/ecs_schemas/1.10.0/ecs_flat.json.gz b/detection_rules/etc/ecs_schemas/1.10.0/ecs_flat.json.gz similarity index 100% rename from etc/ecs_schemas/1.10.0/ecs_flat.json.gz rename to detection_rules/etc/ecs_schemas/1.10.0/ecs_flat.json.gz diff --git a/etc/ecs_schemas/1.10.0/ecs_nested.json.gz b/detection_rules/etc/ecs_schemas/1.10.0/ecs_nested.json.gz similarity index 100% rename from etc/ecs_schemas/1.10.0/ecs_nested.json.gz rename to detection_rules/etc/ecs_schemas/1.10.0/ecs_nested.json.gz diff --git a/etc/ecs_schemas/1.11.0/ecs_flat.json.gz b/detection_rules/etc/ecs_schemas/1.11.0/ecs_flat.json.gz similarity index 100% rename from etc/ecs_schemas/1.11.0/ecs_flat.json.gz rename to detection_rules/etc/ecs_schemas/1.11.0/ecs_flat.json.gz diff --git a/etc/ecs_schemas/1.11.0/ecs_nested.json.gz b/detection_rules/etc/ecs_schemas/1.11.0/ecs_nested.json.gz similarity index 100% rename from etc/ecs_schemas/1.11.0/ecs_nested.json.gz rename to detection_rules/etc/ecs_schemas/1.11.0/ecs_nested.json.gz diff --git a/etc/ecs_schemas/1.12.0/ecs_flat.json.gz b/detection_rules/etc/ecs_schemas/1.12.0/ecs_flat.json.gz similarity index 100% rename from etc/ecs_schemas/1.12.0/ecs_flat.json.gz rename to detection_rules/etc/ecs_schemas/1.12.0/ecs_flat.json.gz diff --git a/etc/ecs_schemas/1.12.0/ecs_nested.json.gz b/detection_rules/etc/ecs_schemas/1.12.0/ecs_nested.json.gz similarity index 100% rename from etc/ecs_schemas/1.12.0/ecs_nested.json.gz rename to detection_rules/etc/ecs_schemas/1.12.0/ecs_nested.json.gz diff --git a/etc/ecs_schemas/1.12.1/ecs_flat.json.gz b/detection_rules/etc/ecs_schemas/1.12.1/ecs_flat.json.gz similarity index 100% rename from etc/ecs_schemas/1.12.1/ecs_flat.json.gz rename to detection_rules/etc/ecs_schemas/1.12.1/ecs_flat.json.gz diff --git a/etc/ecs_schemas/1.12.1/ecs_nested.json.gz b/detection_rules/etc/ecs_schemas/1.12.1/ecs_nested.json.gz similarity index 100% rename from etc/ecs_schemas/1.12.1/ecs_nested.json.gz rename to detection_rules/etc/ecs_schemas/1.12.1/ecs_nested.json.gz diff --git a/etc/ecs_schemas/1.2.0/ecs_flat.json.gz b/detection_rules/etc/ecs_schemas/1.2.0/ecs_flat.json.gz similarity index 100% rename from etc/ecs_schemas/1.2.0/ecs_flat.json.gz rename to detection_rules/etc/ecs_schemas/1.2.0/ecs_flat.json.gz diff --git a/etc/ecs_schemas/1.2.0/ecs_nested.json.gz b/detection_rules/etc/ecs_schemas/1.2.0/ecs_nested.json.gz similarity index 100% rename from etc/ecs_schemas/1.2.0/ecs_nested.json.gz rename to detection_rules/etc/ecs_schemas/1.2.0/ecs_nested.json.gz diff --git a/etc/ecs_schemas/1.3.0/ecs_flat.json.gz b/detection_rules/etc/ecs_schemas/1.3.0/ecs_flat.json.gz similarity index 100% rename from etc/ecs_schemas/1.3.0/ecs_flat.json.gz rename to detection_rules/etc/ecs_schemas/1.3.0/ecs_flat.json.gz diff --git a/etc/ecs_schemas/1.3.0/ecs_nested.json.gz b/detection_rules/etc/ecs_schemas/1.3.0/ecs_nested.json.gz similarity index 100% rename from etc/ecs_schemas/1.3.0/ecs_nested.json.gz rename to detection_rules/etc/ecs_schemas/1.3.0/ecs_nested.json.gz diff --git a/etc/ecs_schemas/1.3.1/ecs_flat.json.gz b/detection_rules/etc/ecs_schemas/1.3.1/ecs_flat.json.gz similarity index 100% rename from etc/ecs_schemas/1.3.1/ecs_flat.json.gz rename to detection_rules/etc/ecs_schemas/1.3.1/ecs_flat.json.gz diff --git a/etc/ecs_schemas/1.3.1/ecs_nested.json.gz b/detection_rules/etc/ecs_schemas/1.3.1/ecs_nested.json.gz similarity index 100% rename from etc/ecs_schemas/1.3.1/ecs_nested.json.gz rename to detection_rules/etc/ecs_schemas/1.3.1/ecs_nested.json.gz diff --git a/etc/ecs_schemas/1.4.0/ecs_flat.json.gz b/detection_rules/etc/ecs_schemas/1.4.0/ecs_flat.json.gz similarity index 100% rename from etc/ecs_schemas/1.4.0/ecs_flat.json.gz rename to detection_rules/etc/ecs_schemas/1.4.0/ecs_flat.json.gz diff --git a/etc/ecs_schemas/1.4.0/ecs_nested.json.gz b/detection_rules/etc/ecs_schemas/1.4.0/ecs_nested.json.gz similarity index 100% rename from etc/ecs_schemas/1.4.0/ecs_nested.json.gz rename to detection_rules/etc/ecs_schemas/1.4.0/ecs_nested.json.gz diff --git a/etc/ecs_schemas/1.5.0/ecs_flat.json.gz b/detection_rules/etc/ecs_schemas/1.5.0/ecs_flat.json.gz similarity index 100% rename from etc/ecs_schemas/1.5.0/ecs_flat.json.gz rename to detection_rules/etc/ecs_schemas/1.5.0/ecs_flat.json.gz diff --git a/etc/ecs_schemas/1.5.0/ecs_nested.json.gz b/detection_rules/etc/ecs_schemas/1.5.0/ecs_nested.json.gz similarity index 100% rename from etc/ecs_schemas/1.5.0/ecs_nested.json.gz rename to detection_rules/etc/ecs_schemas/1.5.0/ecs_nested.json.gz diff --git a/etc/ecs_schemas/1.6.0/ecs_flat.json.gz b/detection_rules/etc/ecs_schemas/1.6.0/ecs_flat.json.gz similarity index 100% rename from etc/ecs_schemas/1.6.0/ecs_flat.json.gz rename to detection_rules/etc/ecs_schemas/1.6.0/ecs_flat.json.gz diff --git a/etc/ecs_schemas/1.6.0/ecs_nested.json.gz b/detection_rules/etc/ecs_schemas/1.6.0/ecs_nested.json.gz similarity index 100% rename from etc/ecs_schemas/1.6.0/ecs_nested.json.gz rename to detection_rules/etc/ecs_schemas/1.6.0/ecs_nested.json.gz diff --git a/etc/ecs_schemas/1.7.0/ecs_flat.json.gz b/detection_rules/etc/ecs_schemas/1.7.0/ecs_flat.json.gz similarity index 100% rename from etc/ecs_schemas/1.7.0/ecs_flat.json.gz rename to detection_rules/etc/ecs_schemas/1.7.0/ecs_flat.json.gz diff --git a/etc/ecs_schemas/1.7.0/ecs_nested.json.gz b/detection_rules/etc/ecs_schemas/1.7.0/ecs_nested.json.gz similarity index 100% rename from etc/ecs_schemas/1.7.0/ecs_nested.json.gz rename to detection_rules/etc/ecs_schemas/1.7.0/ecs_nested.json.gz diff --git a/etc/ecs_schemas/1.8.0/ecs_flat.json.gz b/detection_rules/etc/ecs_schemas/1.8.0/ecs_flat.json.gz similarity index 100% rename from etc/ecs_schemas/1.8.0/ecs_flat.json.gz rename to detection_rules/etc/ecs_schemas/1.8.0/ecs_flat.json.gz diff --git a/etc/ecs_schemas/1.8.0/ecs_nested.json.gz b/detection_rules/etc/ecs_schemas/1.8.0/ecs_nested.json.gz similarity index 100% rename from etc/ecs_schemas/1.8.0/ecs_nested.json.gz rename to detection_rules/etc/ecs_schemas/1.8.0/ecs_nested.json.gz diff --git a/etc/ecs_schemas/1.9.0/ecs_flat.json.gz b/detection_rules/etc/ecs_schemas/1.9.0/ecs_flat.json.gz similarity index 100% rename from etc/ecs_schemas/1.9.0/ecs_flat.json.gz rename to detection_rules/etc/ecs_schemas/1.9.0/ecs_flat.json.gz diff --git a/etc/ecs_schemas/1.9.0/ecs_nested.json.gz b/detection_rules/etc/ecs_schemas/1.9.0/ecs_nested.json.gz similarity index 100% rename from etc/ecs_schemas/1.9.0/ecs_nested.json.gz rename to detection_rules/etc/ecs_schemas/1.9.0/ecs_nested.json.gz diff --git a/etc/ecs_schemas/master_8.2.0.dev/ecs_flat.json.gz b/detection_rules/etc/ecs_schemas/master_8.2.0.dev/ecs_flat.json.gz similarity index 100% rename from etc/ecs_schemas/master_8.2.0.dev/ecs_flat.json.gz rename to detection_rules/etc/ecs_schemas/master_8.2.0.dev/ecs_flat.json.gz diff --git a/etc/lock-multiple.sh b/detection_rules/etc/lock-multiple.sh similarity index 100% rename from etc/lock-multiple.sh rename to detection_rules/etc/lock-multiple.sh diff --git a/etc/non-ecs-schema.json b/detection_rules/etc/non-ecs-schema.json similarity index 100% rename from etc/non-ecs-schema.json rename to detection_rules/etc/non-ecs-schema.json diff --git a/etc/packages.yml b/detection_rules/etc/packages.yml similarity index 100% rename from etc/packages.yml rename to detection_rules/etc/packages.yml diff --git a/etc/rule-mapping.yml b/detection_rules/etc/rule-mapping.yml similarity index 100% rename from etc/rule-mapping.yml rename to detection_rules/etc/rule-mapping.yml diff --git a/etc/rule_template_typosquatting_domain.json b/detection_rules/etc/rule_template_typosquatting_domain.json similarity index 100% rename from etc/rule_template_typosquatting_domain.json rename to detection_rules/etc/rule_template_typosquatting_domain.json diff --git a/etc/security-logo-color-64px.svg b/detection_rules/etc/security-logo-color-64px.svg similarity index 100% rename from etc/security-logo-color-64px.svg rename to detection_rules/etc/security-logo-color-64px.svg diff --git a/etc/stack-schema-map.yaml b/detection_rules/etc/stack-schema-map.yaml similarity index 100% rename from etc/stack-schema-map.yaml rename to detection_rules/etc/stack-schema-map.yaml diff --git a/etc/test_toml.json b/detection_rules/etc/test_toml.json similarity index 100% rename from etc/test_toml.json rename to detection_rules/etc/test_toml.json diff --git a/etc/version.lock.json b/detection_rules/etc/version.lock.json similarity index 100% rename from etc/version.lock.json rename to detection_rules/etc/version.lock.json diff --git a/detection_rules/main.py b/detection_rules/main.py index 4c6f3e337..3539bca08 100644 --- a/detection_rules/main.py +++ b/detection_rules/main.py @@ -250,7 +250,7 @@ def validate_rule(ctx, path): @root.command('validate-all') -def validate_all(fail): +def validate_all(): """Check if all rules validates against a schema.""" RuleCollection.default() click.echo('Rule validation successful') diff --git a/detection_rules/rule_loader.py b/detection_rules/rule_loader.py index d164d7aa8..448f7889e 100644 --- a/detection_rules/rule_loader.py +++ b/detection_rules/rule_loader.py @@ -8,7 +8,8 @@ import io from collections import OrderedDict from dataclasses import dataclass, field from pathlib import Path -from typing import Dict, List, Iterable, Callable, Optional, Union +from subprocess import CalledProcessError +from typing import Callable, Dict, Iterable, List, Optional, Union import click import pytoml @@ -16,9 +17,10 @@ from marshmallow.exceptions import ValidationError from . import utils from .mappings import RtaMappings -from .rule import DeprecatedRule, DeprecatedRuleContents, TOMLRule, TOMLRuleContents +from .rule import (DeprecatedRule, DeprecatedRuleContents, TOMLRule, + TOMLRuleContents) from .schemas import definitions -from .utils import get_path, cached +from .utils import cached, get_path DEFAULT_RULES_DIR = Path(get_path("rules")) DEFAULT_DEPRECATED_DIR = DEFAULT_RULES_DIR / '_deprecated' @@ -98,8 +100,17 @@ def load_locks_from_tag(remote: str, tag: str) -> (str, dict, dict): git(*fetch_tags) commit_hash = git('rev-list', '-1', tag) - version = json.loads(git('show', f'{tag}:etc/version.lock.json')) - deprecated = json.loads(git('show', f'{tag}:etc/deprecated_rules.json')) + try: + version = json.loads(git('show', f'{tag}:detection_rules/etc/version.lock.json')) + except CalledProcessError: + # Adding resiliency to account for the old directory structure + version = json.loads(git('show', f'{tag}:etc/version.lock.json')) + + try: + deprecated = json.loads(git('show', f'{tag}:detection_rules/etc/deprecated_rules.json')) + except CalledProcessError: + # Adding resiliency to account for the old directory structure + deprecated = json.loads(git('show', f'{tag}:etc/deprecated_rules.json')) return commit_hash, version, deprecated @@ -364,10 +375,12 @@ class RuleCollection(BaseCollection): def load_github_pr_rules(labels: list = None, repo: str = 'elastic/detection-rules', token=None, threads=50, verbose=True) -> (Dict[str, TOMLRule], Dict[str, TOMLRule], Dict[str, list]): """Load all rules active as a GitHub PR.""" - import requests - import pytoml from multiprocessing.pool import ThreadPool from pathlib import Path + + import pytoml + import requests + from .ghwrap import GithubClient github = GithubClient(token=token) diff --git a/detection_rules/utils.py b/detection_rules/utils.py index 71963aece..50eefe170 100644 --- a/detection_rules/utils.py +++ b/detection_rules/utils.py @@ -32,7 +32,7 @@ import kql CURR_DIR = os.path.dirname(os.path.abspath(__file__)) ROOT_DIR = os.path.dirname(CURR_DIR) -ETC_DIR = os.path.join(ROOT_DIR, "etc") +ETC_DIR = os.path.join(ROOT_DIR, "detection_rules", "etc") class NonelessDict(dict): @@ -90,29 +90,29 @@ def get_path(*paths) -> str: def get_etc_path(*paths): - """Load a file from the etc/ folder.""" + """Load a file from the detection_rules/etc/ folder.""" return os.path.join(ETC_DIR, *paths) def get_etc_glob_path(*patterns): - """Load a file from the etc/ folder.""" + """Load a file from the detection_rules/etc/ folder.""" pattern = os.path.join(*patterns) return glob.glob(os.path.join(ETC_DIR, pattern)) def get_etc_file(name, mode="r"): - """Load a file from the etc/ folder.""" + """Load a file from the detection_rules/etc/ folder.""" with open(get_etc_path(name), mode) as f: return f.read() def load_etc_dump(*path): - """Load a json/yml/toml file from the etc/ folder.""" + """Load a json/yml/toml file from the detection_rules/etc/ folder.""" return eql.utils.load_dump(get_etc_path(*path)) def save_etc_dump(contents, *path, **kwargs): - """Load a json/yml/toml file from the etc/ folder.""" + """Load a json/yml/toml file from the detection_rules/etc/ folder.""" path = get_etc_path(*path) _, ext = os.path.splitext(path) sort_keys = kwargs.pop('sort_keys', True) diff --git a/docs/deprecating.md b/docs/deprecating.md index 9969d8412..8124e42bf 100644 --- a/docs/deprecating.md +++ b/docs/deprecating.md @@ -1,10 +1,10 @@ # Deprecating rules -Rules that have been version locked (added to [version.lock.json](../etc/version.lock.json)), which also means they +Rules that have been version locked (added to [version.lock.json](../detection_rules/etc/version.lock.json)), which also means they have been added to the detection engine in Kibana, must be properly [deprecated](#steps-to-properly-deprecate-a-rule). If a rule was never version locked (not yet pushed to Kibana or still in non-`production` `maturity`), the rule can -simply be removed with no additional changes, or updated the `maturity = "development"`, which will leave it out of the +simply be removed with no additional changes, or updated the `maturity = "development"`, which will leave it out of the release package to Kibana. @@ -14,7 +14,7 @@ release package to Kibana. 2. Move the rule file to [rules/_deprecated](../rules/_deprecated) 3. Add `deprecation_date` and update `updated_date` to match -Next time the versions are locked, the rule will be added to the [deprecated_rules.json](../etc/deprecated_rules.json) +Next time the versions are locked, the rule will be added to the [deprecated_rules.json](../detection_rules/etc/deprecated_rules.json) file. diff --git a/docs/rule_insights.md b/docs/rule_insights.md index 3e01cd2c8..589d0a91e 100644 --- a/docs/rule_insights.md +++ b/docs/rule_insights.md @@ -11,7 +11,7 @@ The simplest way to index rules from the repo into elasticsearch is to run `python -m detection-rules es index-rules` This will index an enriched version of all rules included and sent to the index `rules-repo--` -- `package-version` is the version defined in `etc/packages.yml` +- `package-version` is the version defined in `detection_rules/etc/packages.yml` - `package hash` is the sha256 hash of the consolidated rules: - sorted by name - flattened @@ -54,7 +54,7 @@ This will generate 2 files under `enriched-rule-indexes/`: - this can be loaded via dev tools or sent as data using curl or any other method that hits the elasticsearch bulk api -The rules are _enriched_ with several pieces of information and so are not identical +The rules are _enriched_ with several pieces of information and so are not identical representations of the rules generated with `view-rule`, though the hashes of the rules are generated before any enrichments are added. @@ -80,7 +80,7 @@ The query can be any valid kql to reduce the scope of included rules, such as ### Importing rules via Kibana -If you have [access](https://www.elastic.co/subscriptions) to machine learning, you can leverage the +If you have [access](https://www.elastic.co/subscriptions) to machine learning, you can leverage the [data-visualizer](https://www.elastic.co/guide/en/kibana/7.11/connect-to-elasticsearch.html#upload-data-kibana) to import the rules via the [importable](#generating-an-index-of-the-rules) file. @@ -88,7 +88,7 @@ to import the rules via the [importable](#generating-an-index-of-the-rules) file ### After the rules have been indexed Once indexed, the rules will need to be added to a [kibana pattern](https://www.elastic.co/guide/en/kibana/7.11/index-patterns.html), -which will then make them searchable via discover or accessible in visualizations. Recommended index pattern is +which will then make them searchable via discover or accessible in visualizations. Recommended index pattern is `rules-*` or `rules-repo-*` diff --git a/tests/test_packages.py b/tests/test_packages.py index 05bba2c56..25ea95626 100644 --- a/tests/test_packages.py +++ b/tests/test_packages.py @@ -50,7 +50,7 @@ class TestPackages(BaseRuleTest): """Test that packages are loading correctly.""" def test_package_loader_default_configs(self): - """Test configs in etc/packages.yml.""" + """Test configs in detection_rules/etc/packages.yml.""" Package.from_config(package_configs) def test_package_summary(self):