diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 2f08af6b3..01b738685 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,15 +1,13 @@ # detection-rules code owners # POC: Elastic Security Intelligence and Analytics Team -tests/**/*.py @brokensound77 @mikaayenson @eric-forte-elastic -detection_rules/ @brokensound77 @mikaayenson @eric-forte-elastic -tests/ @brokensound77 @mikaayenson @eric-forte-elastic -lib/ @brokensound77 @mikaayenson @eric-forte-elastic -rta/ @brokensound77 @mikaayenson @eric-forte-elastic +tests/**/*.py @brokensound77 @mikaayenson @terrancedejesus +detection_rules/ @brokensound77 @mikaayenson @terrancedejesus +tests/ @brokensound77 @mikaayenson @terrancedejesus # skip rta-mapping to avoid the spam -detection_rules/etc/packages.yml @brokensound77 @mikaayenson @eric-forte-elastic -detection_rules/etc/*.json @brokensound77 @mikaayenson @eric-forte-elastic -detection_rules/etc/*.json @brokensound77 @mikaayenson @eric-forte-elastic -detection_rules/etc/*/* @brokensound77 @mikaayenson @eric-forte-elastic +detection_rules/etc/packages.yml @brokensound77 @mikaayenson @terrancedejesus +detection_rules/etc/*.json @brokensound77 @mikaayenson @terrancedejesus +detection_rules/etc/*.json @brokensound77 @mikaayenson @terrancedejesus +detection_rules/etc/*/* @brokensound77 @mikaayenson @terrancedejesus diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index f4247e73f..71331d20d 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -94,7 +94,6 @@ jobs: run: | python -m pip install --upgrade pip pip install .[dev] - pip install lib/kql lib/kibana - name: Prune non-${{matrix.target_branch}} rules env: diff --git a/.github/workflows/get-target-branches.yml b/.github/workflows/get-target-branches.yml index 71fe87f5f..b12661238 100644 --- a/.github/workflows/get-target-branches.yml +++ b/.github/workflows/get-target-branches.yml @@ -16,17 +16,16 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Python 3.12 + - name: Set up Python 3.8 uses: actions/setup-python@v2 with: - python-version: 3.12 + python-version: 3.8 - name: Install dependencies run: | python -m pip install --upgrade pip pip cache purge pip install .[dev] - pip install lib/kql lib/kibana - id: get-branch-list run: | diff --git a/.github/workflows/lock-versions.yml b/.github/workflows/lock-versions.yml index abf0d737a..af33137a1 100644 --- a/.github/workflows/lock-versions.yml +++ b/.github/workflows/lock-versions.yml @@ -26,10 +26,10 @@ jobs: with: fetch-depth: 0 - - name: Set up Python 3.12 + - name: Set up Python 3.8 uses: actions/setup-python@v2 with: - python-version: 3.12 + python-version: 3.8 - name: Install dependencies run: | diff --git a/.github/workflows/manual-backport.yml b/.github/workflows/manual-backport.yml index 28df24fe4..0c7f56144 100644 --- a/.github/workflows/manual-backport.yml +++ b/.github/workflows/manual-backport.yml @@ -50,7 +50,6 @@ jobs: python -m pip install --upgrade pip pip cache purge pip install .[dev] - pip install lib/kql lib/kibana - name: Prune non-"${{github.event.inputs.target_branch}}" rules env: diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index f7b61d27b..b3df19804 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -14,17 +14,16 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Python 3.12 + - name: Set up Python 3.8 uses: actions/setup-python@v2 with: - python-version: 3.12 + python-version: 3.8 - name: Install dependencies run: | python -m pip install --upgrade pip pip cache purge pip install .[dev] - pip install lib/kql lib/kibana - name: Python Lint run: | diff --git a/.github/workflows/release-docs.yml b/.github/workflows/release-docs.yml index 2dc203b28..9af8574bb 100644 --- a/.github/workflows/release-docs.yml +++ b/.github/workflows/release-docs.yml @@ -37,10 +37,10 @@ jobs: fetch-depth: 0 ref: ${{ github.event.inputs.target_branch }} - - name: Set up Python 3.12 + - name: Set up Python 3.8 uses: actions/setup-python@v2 with: - python-version: 3.12 + python-version: 3.8 - name: Install Python dependencies run: | @@ -48,7 +48,6 @@ jobs: python -m pip install --upgrade pip pip cache purge pip install .[dev] - pip install lib/kql lib/kibana - name: Build Integration Docs env: diff --git a/.github/workflows/release-fleet.yml b/.github/workflows/release-fleet.yml index cf4ed411e..bcf26ab1d 100644 --- a/.github/workflows/release-fleet.yml +++ b/.github/workflows/release-fleet.yml @@ -73,10 +73,10 @@ jobs: path: integrations fetch-depth: 0 - - name: Set up Python 3.12 + - name: Set up Python 3.8 uses: actions/setup-python@v2 with: - python-version: 3.12 + python-version: 3.8 - name: Install Python dependencies run: | @@ -84,7 +84,6 @@ jobs: python -m pip install --upgrade pip pip cache purge pip install .[dev] - pip install lib/kql lib/kibana - name: Bump prebuilt rules package version env: diff --git a/Makefile b/Makefile index 92277b631..d61e611ba 100644 --- a/Makefile +++ b/Makefile @@ -5,29 +5,28 @@ VENV := ./env/detection-rules-build VENV_BIN := $(VENV)/bin PYTHON := $(VENV_BIN)/python -PIP := $(VENV_BIN)/pip +PIP := $(VENV_BIN)/python -m pip .PHONY: all all: release + $(VENV): - python3.12 -m pip install --upgrade pip setuptools - python3.12 -m venv $(VENV) + pip3 install virtualenv + virtualenv $(VENV) --python=python3.8 + $(PIP) install .[dev] + $(PIP) install setuptools -U + .PHONY: clean clean: - rm -rf $(VENV) *.egg-info .eggs .egg htmlcov build dist packages .build .tmp .tox __pycache__ lib/kql/build lib/kibana/build lib/kql/*.egg-info lib/kibana/*.egg-info + rm -rf $(VENV) *.egg-info .eggs .egg htmlcov build dist packages .build .tmp .tox __pycache__ .PHONY: deps -deps: $(VENV) install-packages - @echo "Installing all dependencies..." +deps: $(VENV) $(PIP) install .[dev] -.PHONY: install-packages -install-packages: - @echo "Installing kql and kibana packages..." - $(PIP) install lib/kql lib/kibana .PHONY: pytest pytest: $(VENV) deps diff --git a/README.md b/README.md index e20d4d02e..dec6f11d7 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[![Supported Python versions](https://img.shields.io/badge/python-3.12+-yellow.svg)](https://www.python.org/downloads/) +[![Supported Python versions](https://img.shields.io/badge/python-3.8+-yellow.svg)](https://www.python.org/downloads/) [![Unit Tests](https://github.com/elastic/detection-rules/workflows/Unit%20Tests/badge.svg)](https://github.com/elastic/detection-rules/actions) [![Chat](https://img.shields.io/badge/chat-%23security--detection--rules-blueviolet)](https://ela.st/slack) [![ATT&CK navigator coverage](https://img.shields.io/badge/ATT&CK-Navigator-red.svg)](https://ela.st/detection-rules-navigator) @@ -38,29 +38,7 @@ Detection Rules contains more than just static rule files. This repository also ## Getting started -Although rules can be added by manually creating `.toml` files, we don't recommend it. This repository also consists of a python module that aids rule creation and unit testing. Assuming you have Python 3.12+, run the below command to install the dependencies using the makefile: - -```console -✗ make -python3.12 -m pip install --upgrade pip setuptools -Looking in indexes: https://pypi.org/simple -Requirement already satisfied: pip in /opt/homebrew/lib/python3.12/site-packages (24.0) -Requirement already satisfied: setuptools in /opt/homebrew/lib/python3.12/site-packages (69.1.1) -python3.12 -m venv ./env/detection-rules-build -./env/detection-rules-build/bin/pip install --upgrade pip setuptools -Looking in indexes: https://pypi.org/simple -Requirement already satisfied: pip in ./env/detection-rules-build/lib/python3.12/site-packages (24.0) -Collecting setuptools - Using cached setuptools-69.1.1-py3-none-any.whl.metadata (6.2 kB) -Using cached setuptools-69.1.1-py3-none-any.whl (819 kB) -Installing collected packages: setuptools -Successfully installed setuptools-69.1.1 -Installing kql and kibana packages... -... -``` - - -Or install the dependencies using the following command: +Although rules can be added by manually creating `.toml` files, we don't recommend it. This repository also consists of a python module that aids rule creation and unit testing. Assuming you have Python 3.8+, run the below command to install the dependencies: ```console $ pip3 install ".[dev]" Collecting jsl==0.2.4 @@ -75,17 +53,6 @@ Collecting Click==7.0 Downloading Click-7.0-py2.py3-none-any.whl (81 kB) |████████████████████████████████| 81 kB 2.6 MB/s ... -pip3 install packages/kibana packages/kql -``` - -Note: The `kibana` and `kql` packages are not available on PyPI and must be installed from the `packages` directory or `git`. - -```console -pip3 install git+https://github.com/elastic/detection-rules.git#subdirectory=kibana -pip3 install git+https://github.com/elastic/detection-rules.git#subdirectory=kql - -# or locally -pip3 install lib/kibana lib/kql ``` To confirm that everything was properly installed, run with the `--help` flag diff --git a/detection_rules/__init__.py b/detection_rules/__init__.py index bd75c6f42..810fd2865 100644 --- a/detection_rules/__init__.py +++ b/detection_rules/__init__.py @@ -8,7 +8,7 @@ import sys -assert (3, 12) <= sys.version_info < (4, 0), "Only Python 3.12+ supported" +assert (3, 8) <= sys.version_info < (4, 0), "Only Python 3.8+ supported" from . import ( # noqa: E402 devtools, diff --git a/detection_rules/__main__.py b/detection_rules/__main__.py index e045df810..b50fc3e92 100644 --- a/detection_rules/__main__.py +++ b/detection_rules/__main__.py @@ -10,7 +10,7 @@ import sys import click -assert (3, 12) <= sys.version_info < (4, 0), "Only Python 3.12+ supported" +assert (3, 8) <= sys.version_info < (4, 0), "Only Python 3.8+ supported" from .main import root # noqa: E402 diff --git a/detection_rules/devtools.py b/detection_rules/devtools.py index e0163ae70..1c9a5c5bb 100644 --- a/detection_rules/devtools.py +++ b/detection_rules/devtools.py @@ -1255,7 +1255,7 @@ def build_integration_schemas(overwrite: bool, integration: str): else: build_integrations_schemas(overwrite=overwrite) end_time = time.perf_counter() - click.echo(f"Time taken to generate schemas: {(end_time - start_time) / 60:.2f} minutes") + click.echo(f"Time taken to generate schemas: {(end_time - start_time)/60:.2f} minutes") @integrations_group.command('show-latest-compatible') diff --git a/detection_rules/misc.py b/detection_rules/misc.py index f5f2cb4b2..82d8e4893 100644 --- a/detection_rules/misc.py +++ b/detection_rules/misc.py @@ -149,10 +149,10 @@ def schema_prompt(name, value=None, is_required=False, **options): if enum and _val not in enum: print('{} not in valid options: {}'.format(_val, ', '.join(enum))) return False - if minimum and (type(_val) is int and int(_val) < minimum): + if minimum and (type(_val) == int and int(_val) < minimum): print('{} is less than the minimum: {}'.format(str(_val), str(minimum))) return False - if maximum and (type(_val) is int and int(_val) > maximum): + if maximum and (type(_val) == int and int(_val) > maximum): print('{} is greater than the maximum: {}'.format(str(_val), str(maximum))) return False if field_type == 'boolean' and _val.lower() not in ('true', 'false'): @@ -161,7 +161,7 @@ def schema_prompt(name, value=None, is_required=False, **options): return True def _convert_type(_val): - if field_type == 'boolean' and not type(_val) is bool: + if field_type == 'boolean' and not type(_val) == bool: _val = True if _val.lower() == 'true' else False return int(_val) if field_type in ('number', 'integer') else _val diff --git a/detection_rules/utils.py b/detection_rules/utils.py index 6bc7e527f..19e265170 100644 --- a/detection_rules/utils.py +++ b/detection_rules/utils.py @@ -6,6 +6,7 @@ """Util functions.""" import base64 import contextlib +import distutils.spawn import functools import glob import gzip @@ -58,7 +59,7 @@ def gopath() -> Optional[str]: if env_path: return env_path - go_bin = shutil.which("go") + go_bin = distutils.spawn.find_executable("go") if go_bin: output = subprocess.check_output([go_bin, "env"], encoding="utf-8").splitlines() for line in output: diff --git a/lib/kibana/kibana/__init__.py b/kibana/__init__.py similarity index 95% rename from lib/kibana/kibana/__init__.py rename to kibana/__init__.py index d96615c4c..2e174fa04 100644 --- a/lib/kibana/kibana/__init__.py +++ b/kibana/__init__.py @@ -8,7 +8,6 @@ from .connector import Kibana from .resources import RuleResource, Signal -__version__ = '0.1.0' __all__ = ( "Kibana", "RuleResource", diff --git a/lib/kibana/kibana/connector.py b/kibana/connector.py similarity index 100% rename from lib/kibana/kibana/connector.py rename to kibana/connector.py diff --git a/lib/kibana/kibana/resources.py b/kibana/resources.py similarity index 100% rename from lib/kibana/kibana/resources.py rename to kibana/resources.py diff --git a/lib/kql/kql/__init__.py b/kql/__init__.py similarity index 100% rename from lib/kql/kql/__init__.py rename to kql/__init__.py diff --git a/lib/kql/kql/ast.py b/kql/ast.py similarity index 100% rename from lib/kql/kql/ast.py rename to kql/ast.py diff --git a/lib/kql/kql/dsl.py b/kql/dsl.py similarity index 100% rename from lib/kql/kql/dsl.py rename to kql/dsl.py diff --git a/lib/kql/kql/eql2kql.py b/kql/eql2kql.py similarity index 100% rename from lib/kql/kql/eql2kql.py rename to kql/eql2kql.py diff --git a/lib/kql/kql/errors.py b/kql/errors.py similarity index 100% rename from lib/kql/kql/errors.py rename to kql/errors.py diff --git a/lib/kql/kql/evaluator.py b/kql/evaluator.py similarity index 100% rename from lib/kql/kql/evaluator.py rename to kql/evaluator.py diff --git a/lib/kql/kql/kql.g b/kql/kql.g similarity index 100% rename from lib/kql/kql/kql.g rename to kql/kql.g diff --git a/lib/kql/kql/kql2eql.py b/kql/kql2eql.py similarity index 100% rename from lib/kql/kql/kql2eql.py rename to kql/kql2eql.py diff --git a/lib/kql/kql/optimizer.py b/kql/optimizer.py similarity index 100% rename from lib/kql/kql/optimizer.py rename to kql/optimizer.py diff --git a/lib/kql/kql/parser.py b/kql/parser.py similarity index 100% rename from lib/kql/kql/parser.py rename to kql/parser.py diff --git a/lib/kibana/pyproject.toml b/lib/kibana/pyproject.toml deleted file mode 100644 index bb88beb4d..000000000 --- a/lib/kibana/pyproject.toml +++ /dev/null @@ -1,28 +0,0 @@ -[project] -name = "detection-rules-kibana" -version = "0.1.0" -description = "Kibana API utilities for Elastic Detection Rules" -license = {text = "Elastic License v2"} -keywords = ["Elastic", "Kibana", "Detection Rules", "Security", "Elasticsearch"] -classifiers = [ - "Intended Audience :: Developers", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.12", - "Topic :: Security", - "Topic :: Software Development :: Build Tools", - "Topic :: Software Development :: Libraries :: Python Modules", - "Topic :: Software Development", -] -requires-python = ">=3.12" -dependencies = [ - "requests>=2.25,<3.0", - "elasticsearch~=8.1", -] - -[project.urls] -Homepage = "https://github.com/elastic/detection-rules" -License = "https://github.com/elastic/detection-rules/blob/main/LICENSE.txt" - -[build-system] -requires = ["setuptools", "wheel"] -build-backend = "setuptools.build_meta" diff --git a/lib/kql/pyproject.toml b/lib/kql/pyproject.toml deleted file mode 100644 index 9bf1f577c..000000000 --- a/lib/kql/pyproject.toml +++ /dev/null @@ -1,31 +0,0 @@ -[project] -name = "detection-rules-kql" -version = "0.1.6" -description = "Kibana Query Language parser for Elastic Detection Rules" -license = {text = "Elastic License v2"} -keywords = ["Elastic", "sour", "Detection Rules", "Security", "Elasticsearch", "kql"] -classifiers = [ - "Intended Audience :: Developers", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.12", - "Topic :: Security", - "Topic :: Software Development :: Build Tools", - "Topic :: Software Development :: Libraries :: Python Modules", - "Topic :: Software Development", -] -requires-python = ">=3.12" -dependencies = [ - "eql==0.9.19", - "lark-parser>=0.11.1", -] - -[project.urls] -Homepage = "https://github.com/elastic/detection-rules" -License = "https://github.com/elastic/detection-rules/blob/main/LICENSE.txt" - -[build-system] -requires = ["setuptools", "wheel"] -build-backend = "setuptools.build_meta" - -[tool.setuptools.package-data] -kql = ["*.g"] diff --git a/pyproject.toml b/pyproject.toml index dee0fb8c0..bf5ec9f58 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,14 +3,17 @@ name = "detection_rules" version = "0.1.0" description = "Detection Rules is the home for rules used by Elastic Security. This repository is used for the development, maintenance, testing, validation, and release of rules for Elastic Security’s Detection Engine." readme = "README.md" -requires-python = ">=3.12" +requires-python = ">=3.8" license = {file = "LICENSE.txt"} keywords = ["Detection Rules", "Continuous Monitoring", "Data Protection", "Elastic", "Elastic Endgame", "Endpoint Security"] classifiers = [ "Topic :: Software Development :: Build Tools", "Operating System :: OS Independent", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python", "Topic :: Security", "Topic :: Software Development :: Libraries :: Python Modules", "Topic :: Software Development :: Libraries", @@ -31,16 +34,18 @@ dependencies = [ "marshmallow~=3.13.0", "pywin32 ; platform_system=='Windows'", "pytoml==0.1.21", - "PyYAML~=6.0.1", + "PyYAML~=5.3 ; python_version<='3.9'", + "PyYAML~=6.0.1 ; python_version>='3.10'", "requests~=2.27", "toml==0.10.0", "typing-inspect==0.8.0", - "typing-extensions==4.8.0", + "typing-extensions==4.5.0 ; python_version<='3.11'", + "typing-extensions==4.8.0 ; python_version>='3.12'", "XlsxWriter~=1.3.6", "semver==3.0.0-dev.4" ] [project.optional-dependencies] -dev = ["pep8-naming==0.7.0", "PyGithub==1.55", "flake8==7.0.0", "pyflakes==3.2.0", "pytest>=3.6", "pre-commit==2.20.0"] +dev = ["pep8-naming==0.7.0", "PyGithub==1.55", "flake8==3.8.1", "pyflakes==2.2.0", "pytest>=3.6", "pre-commit==2.20.0"] [project.urls] "Homepage" = "https://github.com/elastic/detection-rules" @@ -50,7 +55,7 @@ dev = ["pep8-naming==0.7.0", "PyGithub==1.55", "flake8==7.0.0", "pyflakes==3.2.0 [tool.setuptools] package-data = {"kql" = ["*.g"]} -packages = ["detection_rules", "rta"] +packages = ["detection_rules", "kql", "kibana", "rta"] [tool.pytest.ini_options] filterwarnings = [