security-tools/sigma-rules
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Revert "[FR] Independently package kql / kibana and bump to py3.12 (#3492)"

Browse Source 
This reverts commit fc139fc3c2.
This commit is contained in:
Mika Ayenson
2024-03-14 19:48:50 -05:00
parent fc139fc3c2
commit 3d2a36be32
31 changed files with 48 additions and 144 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/CODEOWNERS
+7 -9
View File
@@ -1,15 +1,13 @@
# detection-rules code owners
# POC: Elastic Security Intelligence and Analytics Team
tests/**/*.py @brokensound77 @mikaayenson @eric-forte-elastic
detection_rules/ @brokensound77 @mikaayenson @eric-forte-elastic
tests/ @brokensound77 @mikaayenson @eric-forte-elastic
lib/ @brokensound77 @mikaayenson @eric-forte-elastic
rta/ @brokensound77 @mikaayenson @eric-forte-elastic
tests/**/*.py @brokensound77 @mikaayenson @terrancedejesus
detection_rules/ @brokensound77 @mikaayenson @terrancedejesus
tests/ @brokensound77 @mikaayenson @terrancedejesus
# skip rta-mapping to avoid the spam
detection_rules/etc/packages.yml @brokensound77 @mikaayenson @eric-forte-elastic
detection_rules/etc/*.json @brokensound77 @mikaayenson @eric-forte-elastic
detection_rules/etc/*.json @brokensound77 @mikaayenson @eric-forte-elastic
detection_rules/etc/*/* @brokensound77 @mikaayenson @eric-forte-elastic
detection_rules/etc/packages.yml @brokensound77 @mikaayenson @terrancedejesus
detection_rules/etc/*.json @brokensound77 @mikaayenson @terrancedejesus
detection_rules/etc/*.json @brokensound77 @mikaayenson @terrancedejesus
detection_rules/etc/*/* @brokensound77 @mikaayenson @terrancedejesus
.github/workflows/backport.yml
-1
View File
@@ -94,7 +94,6 @@ jobs:
run: |
python -m pip install --upgrade pip
pip install .[dev]
pip install lib/kql lib/kibana
- name: Prune non-${{matrix.target_branch}} rules
env:
.github/workflows/get-target-branches.yml
+2 -3
View File
@@ -16,17 +16,16 @@ jobs:
steps:
- uses: actions/checkout@v2
- name: Set up Python 3.12
- name: Set up Python 3.8
uses: actions/setup-python@v2
with:
python-version: 3.12
python-version: 3.8
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip cache purge
pip install .[dev]
pip install lib/kql lib/kibana
- id: get-branch-list
run: |
.github/workflows/lock-versions.yml
+2 -2
View File
@@ -26,10 +26,10 @@ jobs:
with:
fetch-depth: 0
- name: Set up Python 3.12
- name: Set up Python 3.8
uses: actions/setup-python@v2
with:
python-version: 3.12
python-version: 3.8
- name: Install dependencies
run: |
.github/workflows/manual-backport.yml
-1
View File
@@ -50,7 +50,6 @@ jobs:
python -m pip install --upgrade pip
pip cache purge
pip install .[dev]
pip install lib/kql lib/kibana
- name: Prune non-"${{github.event.inputs.target_branch}}" rules
env:
.github/workflows/pythonpackage.yml
+2 -3
View File
@@ -14,17 +14,16 @@ jobs:
steps:
- uses: actions/checkout@v2
- name: Set up Python 3.12
- name: Set up Python 3.8
uses: actions/setup-python@v2
with:
python-version: 3.12
python-version: 3.8
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip cache purge
pip install .[dev]
pip install lib/kql lib/kibana
- name: Python Lint
run: |
.github/workflows/release-docs.yml
+2 -3
View File
@@ -37,10 +37,10 @@ jobs:
fetch-depth: 0
ref: ${{ github.event.inputs.target_branch }}
- name: Set up Python 3.12
- name: Set up Python 3.8
uses: actions/setup-python@v2
with:
python-version: 3.12
python-version: 3.8
- name: Install Python dependencies
run: |
@@ -48,7 +48,6 @@ jobs:
python -m pip install --upgrade pip
pip cache purge
pip install .[dev]
pip install lib/kql lib/kibana
- name: Build Integration Docs
env:
.github/workflows/release-fleet.yml
+2 -3
View File
@@ -73,10 +73,10 @@ jobs:
path: integrations
fetch-depth: 0
- name: Set up Python 3.12
- name: Set up Python 3.8
uses: actions/setup-python@v2
with:
python-version: 3.12
python-version: 3.8
- name: Install Python dependencies
run: |
@@ -84,7 +84,6 @@ jobs:
python -m pip install --upgrade pip
pip cache purge
pip install .[dev]
pip install lib/kql lib/kibana
- name: Bump prebuilt rules package version
env:
Makefile
+9 -10
View File
@@ -5,29 +5,28 @@
VENV := ./env/detection-rules-build
VENV_BIN := $(VENV)/bin
PYTHON := $(VENV_BIN)/python
PIP := $(VENV_BIN)/pip
PIP := $(VENV_BIN)/python -m pip
.PHONY: all
all: release
$(VENV):
python3.12 -m pip install --upgrade pip setuptools
python3.12 -m venv $(VENV)
pip3 install virtualenv
virtualenv $(VENV) --python=python3.8
$(PIP) install .[dev]
$(PIP) install setuptools -U
.PHONY: clean
clean:
rm -rf $(VENV) *.egg-info .eggs .egg htmlcov build dist packages .build .tmp .tox __pycache__ lib/kql/build lib/kibana/build lib/kql/*.egg-info lib/kibana/*.egg-info
rm -rf $(VENV) *.egg-info .eggs .egg htmlcov build dist packages .build .tmp .tox __pycache__
.PHONY: deps
deps: $(VENV) install-packages
@echo "Installing all dependencies..."
deps: $(VENV)
$(PIP) install .[dev]
.PHONY: install-packages
install-packages:
@echo "Installing kql and kibana packages..."
$(PIP) install lib/kql lib/kibana
.PHONY: pytest
pytest: $(VENV) deps
README.md
+2 -35
View File
@@ -1,4 +1,4 @@
[![Supported Python versions](https://img.shields.io/badge/python-3.12+-yellow.svg)](https://www.python.org/downloads/)
[![Supported Python versions](https://img.shields.io/badge/python-3.8+-yellow.svg)](https://www.python.org/downloads/)
[![Unit Tests](https://github.com/elastic/detection-rules/workflows/Unit%20Tests/badge.svg)](https://github.com/elastic/detection-rules/actions)
[![Chat](https://img.shields.io/badge/chat-%23security--detection--rules-blueviolet)](https://ela.st/slack)
[![ATT&CK navigator coverage](https://img.shields.io/badge/ATT&CK-Navigator-red.svg)](https://ela.st/detection-rules-navigator)
@@ -38,29 +38,7 @@ Detection Rules contains more than just static rule files. This repository also
## Getting started
Although rules can be added by manually creating `.toml` files, we don't recommend it. This repository also consists of a python module that aids rule creation and unit testing. Assuming you have Python 3.12+, run the below command to install the dependencies using the makefile:
```console
✗ make
python3.12 -m pip install --upgrade pip setuptools
Looking in indexes: https://pypi.org/simple
Requirement already satisfied: pip in /opt/homebrew/lib/python3.12/site-packages (24.0)
Requirement already satisfied: setuptools in /opt/homebrew/lib/python3.12/site-packages (69.1.1)
python3.12 -m venv ./env/detection-rules-build
./env/detection-rules-build/bin/pip install --upgrade pip setuptools
Looking in indexes: https://pypi.org/simple
Requirement already satisfied: pip in ./env/detection-rules-build/lib/python3.12/site-packages (24.0)
Collecting setuptools
Using cached setuptools-69.1.1-py3-none-any.whl.metadata (6.2 kB)
Using cached setuptools-69.1.1-py3-none-any.whl (819 kB)
Installing collected packages: setuptools
Successfully installed setuptools-69.1.1
Installing kql and kibana packages...
...
```
Or install the dependencies using the following command:
Although rules can be added by manually creating `.toml` files, we don't recommend it. This repository also consists of a python module that aids rule creation and unit testing. Assuming you have Python 3.8+, run the below command to install the dependencies:
```console
$ pip3 install ".[dev]"
Collecting jsl==0.2.4
@@ -75,17 +53,6 @@ Collecting Click==7.0
Downloading Click-7.0-py2.py3-none-any.whl (81 kB)
|████████████████████████████████| 81 kB 2.6 MB/s
...
pip3 install packages/kibana packages/kql
```
Note: The `kibana` and `kql` packages are not available on PyPI and must be installed from the `packages` directory or `git`.
```console
pip3 install git+https://github.com/elastic/detection-rules.git#subdirectory=kibana
pip3 install git+https://github.com/elastic/detection-rules.git#subdirectory=kql
# or locally
pip3 install lib/kibana lib/kql
```
To confirm that everything was properly installed, run with the `--help` flag
detection_rules/__init__.py
+1 -1
View File
@@ -8,7 +8,7 @@
import sys
assert (3, 12) <= sys.version_info < (4, 0), "Only Python 3.12+ supported"
assert (3, 8) <= sys.version_info < (4, 0), "Only Python 3.8+ supported"
from . import ( # noqa: E402
devtools,
detection_rules/__main__.py
+1 -1
View File
@@ -10,7 +10,7 @@ import sys
import click
assert (3, 12) <= sys.version_info < (4, 0), "Only Python 3.12+ supported"
assert (3, 8) <= sys.version_info < (4, 0), "Only Python 3.8+ supported"
from .main import root # noqa: E402
detection_rules/devtools.py
+1 -1
View File
@@ -1255,7 +1255,7 @@ def build_integration_schemas(overwrite: bool, integration: str):
else:
build_integrations_schemas(overwrite=overwrite)
end_time = time.perf_counter()
click.echo(f"Time taken to generate schemas: {(end_time - start_time) / 60:.2f} minutes")
click.echo(f"Time taken to generate schemas: {(end_time - start_time)/60:.2f} minutes")
@integrations_group.command('show-latest-compatible')
detection_rules/misc.py
+3 -3
View File
@@ -149,10 +149,10 @@ def schema_prompt(name, value=None, is_required=False, **options):
if enum and _val not in enum:
print('{} not in valid options: {}'.format(_val, ', '.join(enum)))
return False
if minimum and (type(_val) is int and int(_val) < minimum):
if minimum and (type(_val) == int and int(_val) < minimum):
print('{} is less than the minimum: {}'.format(str(_val), str(minimum)))
return False
if maximum and (type(_val) is int and int(_val) > maximum):
if maximum and (type(_val) == int and int(_val) > maximum):
print('{} is greater than the maximum: {}'.format(str(_val), str(maximum)))
return False
if field_type == 'boolean' and _val.lower() not in ('true', 'false'):
@@ -161,7 +161,7 @@ def schema_prompt(name, value=None, is_required=False, **options):
return True
def _convert_type(_val):
if field_type == 'boolean' and not type(_val) is bool:
if field_type == 'boolean' and not type(_val) == bool:
_val = True if _val.lower() == 'true' else False
return int(_val) if field_type in ('number', 'integer') else _val
detection_rules/utils.py
+2 -1
View File
@@ -6,6 +6,7 @@
"""Util functions."""
import base64
import contextlib
import distutils.spawn
import functools
import glob
import gzip
@@ -58,7 +59,7 @@ def gopath() -> Optional[str]:
if env_path:
return env_path
go_bin = shutil.which("go")
go_bin = distutils.spawn.find_executable("go")
if go_bin:
output = subprocess.check_output([go_bin, "env"], encoding="utf-8").splitlines()
for line in output:
lib/kibana/kibana/__init__.py → kibana/__init__.py
-1
View File
@@ -8,7 +8,6 @@
from .connector import Kibana
from .resources import RuleResource, Signal
__version__ = '0.1.0'
__all__ = (
"Kibana",
"RuleResource",
lib/kibana/kibana/connector.py → kibana/connector.py
View File
lib/kibana/kibana/resources.py → kibana/resources.py
View File
lib/kql/kql/__init__.py → kql/__init__.py
View File
lib/kql/kql/ast.py → kql/ast.py
View File
lib/kql/kql/dsl.py → kql/dsl.py
View File
lib/kql/kql/eql2kql.py → kql/eql2kql.py
View File
lib/kql/kql/errors.py → kql/errors.py
View File
lib/kql/kql/evaluator.py → kql/evaluator.py
View File
lib/kql/kql/kql.g → kql/kql.g
View File
lib/kql/kql/kql2eql.py → kql/kql2eql.py
View File
lib/kql/kql/optimizer.py → kql/optimizer.py
View File
lib/kql/kql/parser.py → kql/parser.py
View File
lib/kibana/pyproject.toml
-28
View File
@@ -1,28 +0,0 @@
[project]
name = "detection-rules-kibana"
version = "0.1.0"
description = "Kibana API utilities for Elastic Detection Rules"
license = {text = "Elastic License v2"}
keywords = ["Elastic", "Kibana", "Detection Rules", "Security", "Elasticsearch"]
classifiers = [
"Intended Audience :: Developers",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.12",
"Topic :: Security",
"Topic :: Software Development :: Build Tools",
"Topic :: Software Development :: Libraries :: Python Modules",
"Topic :: Software Development",
]
requires-python = ">=3.12"
dependencies = [
"requests>=2.25,<3.0",
"elasticsearch~=8.1",
]
[project.urls]
Homepage = "https://github.com/elastic/detection-rules"
License = "https://github.com/elastic/detection-rules/blob/main/LICENSE.txt"
[build-system]
requires = ["setuptools", "wheel"]
build-backend = "setuptools.build_meta"
lib/kql/pyproject.toml
-31
View File
@@ -1,31 +0,0 @@
[project]
name = "detection-rules-kql"
version = "0.1.6"
description = "Kibana Query Language parser for Elastic Detection Rules"
license = {text = "Elastic License v2"}
keywords = ["Elastic", "sour", "Detection Rules", "Security", "Elasticsearch", "kql"]
classifiers = [
"Intended Audience :: Developers",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.12",
"Topic :: Security",
"Topic :: Software Development :: Build Tools",
"Topic :: Software Development :: Libraries :: Python Modules",
"Topic :: Software Development",
]
requires-python = ">=3.12"
dependencies = [
"eql==0.9.19",
"lark-parser>=0.11.1",
]
[project.urls]
Homepage = "https://github.com/elastic/detection-rules"
License = "https://github.com/elastic/detection-rules/blob/main/LICENSE.txt"
[build-system]
requires = ["setuptools", "wheel"]
build-backend = "setuptools.build_meta"
[tool.setuptools.package-data]
kql = ["*.g"]
pyproject.toml
+12 -7
View File
@@ -3,14 +3,17 @@ name = "detection_rules"
version = "0.1.0"
description = "Detection Rules is the home for rules used by Elastic Security. This repository is used for the development, maintenance, testing, validation, and release of rules for Elastic Security’s Detection Engine."
readme = "README.md"
requires-python = ">=3.12"
requires-python = ">=3.8"
license = {file = "LICENSE.txt"}
keywords = ["Detection Rules", "Continuous Monitoring", "Data Protection", "Elastic", "Elastic Endgame", "Endpoint Security"]
classifiers = [
"Topic :: Software Development :: Build Tools",
"Operating System :: OS Independent",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.12",
"Programming Language :: Python :: 3.8",
"Programming Language :: Python :: 3.9",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
"Programming Language :: Python",
"Topic :: Security",
"Topic :: Software Development :: Libraries :: Python Modules",
"Topic :: Software Development :: Libraries",
@@ -31,16 +34,18 @@ dependencies = [
"marshmallow~=3.13.0",
"pywin32 ; platform_system=='Windows'",
"pytoml==0.1.21",
"PyYAML~=6.0.1",
"PyYAML~=5.3 ; python_version<='3.9'",
"PyYAML~=6.0.1 ; python_version>='3.10'",
"requests~=2.27",
"toml==0.10.0",
"typing-inspect==0.8.0",
"typing-extensions==4.8.0",
"typing-extensions==4.5.0 ; python_version<='3.11'",
"typing-extensions==4.8.0 ; python_version>='3.12'",
"XlsxWriter~=1.3.6",
"semver==3.0.0-dev.4"
]
[project.optional-dependencies]
dev = ["pep8-naming==0.7.0", "PyGithub==1.55", "flake8==7.0.0", "pyflakes==3.2.0", "pytest>=3.6", "pre-commit==2.20.0"]
dev = ["pep8-naming==0.7.0", "PyGithub==1.55", "flake8==3.8.1", "pyflakes==2.2.0", "pytest>=3.6", "pre-commit==2.20.0"]
[project.urls]
"Homepage" = "https://github.com/elastic/detection-rules"
@@ -50,7 +55,7 @@ dev = ["pep8-naming==0.7.0", "PyGithub==1.55", "flake8==7.0.0", "pyflakes==3.2.0
[tool.setuptools]
package-data = {"kql" = ["*.g"]}
packages = ["detection_rules", "rta"]
packages = ["detection_rules", "kql", "kibana", "rta"]
[tool.pytest.ini_options]
filterwarnings = [