From 5c3523954eb32658666e9660e76c403e30632fe2 Mon Sep 17 00:00:00 2001 From: Mika Ayenson Date: Tue, 19 Mar 2024 14:07:16 -0500 Subject: [PATCH] [FR] Update Python Dependency Versions (#3515) --- .github/workflows/pythonpackage.yml | 2 +- Makefile | 12 ++++- detection_rules/etc/test_cli.bash | 56 ++++++++++++++++++++++++ detection_rules/etc/test_remote_cli.bash | 15 +++++++ detection_rules/misc.py | 17 ++++--- lib/kibana/pyproject.toml | 2 +- lib/kql/pyproject.toml | 2 +- pyproject.toml | 28 ++++++------ 8 files changed, 109 insertions(+), 25 deletions(-) create mode 100755 detection_rules/etc/test_cli.bash create mode 100755 detection_rules/etc/test_remote_cli.bash diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index ec915aaa9..8a787f9ab 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -28,7 +28,7 @@ jobs: - name: Python Lint run: | - python -m flake8 tests detection_rules --ignore D203 --max-line-length 120 + python -m flake8 tests detection_rules --ignore D203,N815 --max-line-length 120 - name: Python License Check run: | diff --git a/Makefile b/Makefile index 92277b631..6c1025479 100644 --- a/Makefile +++ b/Makefile @@ -41,11 +41,21 @@ license-check: $(VENV) deps .PHONY: lint lint: $(VENV) deps @echo "LINTING" - $(PYTHON) -m flake8 tests detection_rules --ignore D203 --max-line-length 120 + $(PYTHON) -m flake8 tests detection_rules --ignore D203,N815 --max-line-length 120 .PHONY: test test: $(VENV) lint pytest +.PHONY: test-cli +test-cli: $(VENV) + @echo "Executing test_cli script..." + @./detection_rules/etc/test_cli.bash + +.PHONY: test-remote-cli +test-cli: $(VENV) + @echo "Executing test_remote_cli script..." + @./detection_rules/etc/test_remote_cli.bash + .PHONY: release release: deps @echo "RELEASE: $(app_name)" diff --git a/detection_rules/etc/test_cli.bash b/detection_rules/etc/test_cli.bash new file mode 100755 index 000000000..87717cc7b --- /dev/null +++ b/detection_rules/etc/test_cli.bash @@ -0,0 +1,56 @@ +#!/bin/bash + +# Path to the virtual environment +VENV_PATH="./env/detection-rules-build" + +# Activate the virtual environment +source "$VENV_PATH/bin/activate" + +echo "Running detection-rules CLI tests..." + +echo "Refreshing redirect mappings in ATT&CK" +python -m detection_rules dev attack refresh-redirect-mappings + +echo "Viewing rule: threat_intel_indicator_match_address.toml" +python -m detection_rules view-rule rules/cross-platform/threat_intel_indicator_match_address.toml + +echo "Exporting rule by ID: 0a97b20f-4144-49ea-be32-b540ecc445de" +python -m detection_rules export-rules --rule-id 0a97b20f-4144-49ea-be32-b540ecc445de + +echo "Updating rule data schemas" +python -m detection_rules dev schemas update-rule-data + +echo "Validating rule: execution_github_new_event_action_for_pat.toml" +python -m detection_rules validate-rule rules_building_block/execution_github_new_event_action_for_pat.toml + +echo "Checking licenses" +python -m detection_rules dev license-check + +echo "Building release and updating version lock" +python -m detection_rules dev build-release --update-version-lock + +echo "Refreshing ATT&CK data" +python -m detection_rules dev attack refresh-data + +echo "Updating rules with latest ATT&CK data" +python -m detection_rules dev attack update-rules + +echo "Getting target branches" +python -m detection_rules dev utils get-branches + +echo "Showing latest compatible version for security_detection_engine with stack version 8.12.0" +python -m detection_rules dev integrations show-latest-compatible --package endpoint --stack_version 8.12.0 + +echo "Building limited rules for stack version 8.12" +python -m detection_rules build-limited-rules --stack-version "8.12" --output-file "output_file.ndjson" + +echo "Building limited rules for stack version 8.12 with custom rules" +python -m detection_rules generate-rules-index --overwrite + +echo "Building manifests for integrations" +python -m detection_rules dev integrations build-manifests -i endpoint + +echo "Building schemas for integrations" +python -m detection_rules dev integrations build-schemas -i endpoint + +echo "Detection-rules CLI tests completed!" diff --git a/detection_rules/etc/test_remote_cli.bash b/detection_rules/etc/test_remote_cli.bash new file mode 100755 index 000000000..2d9eccc63 --- /dev/null +++ b/detection_rules/etc/test_remote_cli.bash @@ -0,0 +1,15 @@ +#!/bin/bash + +# Path to the virtual environment +VENV_PATH="./env/detection-rules-build" + +# Activate the virtual environment +source "$VENV_PATH/bin/activate" + +echo "Running detection-rules remote CLI tests..." + +echo "Performing a quick rule alerts search..." +echo "Requires .detection-rules-cfg.json credentials file set." +python -m detection_rules kibana search-alerts + +echo "Detection-rules CLI tests completed!" diff --git a/detection_rules/misc.py b/detection_rules/misc.py index f5f2cb4b2..26c190c6c 100644 --- a/detection_rules/misc.py +++ b/detection_rules/misc.py @@ -431,13 +431,16 @@ def add_client(*client_type, add_to_ctx=True, add_func_arg=True): if 'kibana' in client_type: # for nested ctx invocation, no need to re-auth if an existing client is already passed kibana_client: Kibana = kwargs.get('kibana_client') - try: - with kibana_client: - if kibana_client and isinstance(kibana_client, Kibana) and kibana_client.version: - pass - else: - kibana_client = get_kibana_client(**kibana_client_args) - except (requests.HTTPError, AttributeError): + if kibana_client and isinstance(kibana_client, Kibana): + + try: + with kibana_client: + if kibana_client.version: + pass # kibana_client is valid and can be used directly + except (requests.HTTPError, AttributeError): + kibana_client = get_kibana_client(**kibana_client_args) + else: + # Instantiate a new Kibana client if none was provided or if the provided one is not usable kibana_client = get_kibana_client(**kibana_client_args) if add_func_arg: diff --git a/lib/kibana/pyproject.toml b/lib/kibana/pyproject.toml index bb88beb4d..6565fe529 100644 --- a/lib/kibana/pyproject.toml +++ b/lib/kibana/pyproject.toml @@ -16,7 +16,7 @@ classifiers = [ requires-python = ">=3.12" dependencies = [ "requests>=2.25,<3.0", - "elasticsearch~=8.1", + "elasticsearch~=8.12.1", ] [project.urls] diff --git a/lib/kql/pyproject.toml b/lib/kql/pyproject.toml index 9bf1f577c..87752574e 100644 --- a/lib/kql/pyproject.toml +++ b/lib/kql/pyproject.toml @@ -16,7 +16,7 @@ classifiers = [ requires-python = ">=3.12" dependencies = [ "eql==0.9.19", - "lark-parser>=0.11.1", + "lark-parser>=0.12.0", ] [project.urls] diff --git a/pyproject.toml b/pyproject.toml index dee0fb8c0..c07a6ceef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,28 +19,28 @@ classifiers = [ "Topic :: Utilities" ] dependencies = [ - "Click~=8.1.0", - "elasticsearch~=8.1", + "Click~=8.1.7", + "elasticsearch~=8.12.1", "eql==0.9.19", "jsl==0.2.4", - "jsonschema>=3.2.0", - "marko==2.0.1", - "marshmallow-dataclass[union]~=8.5.12", - "marshmallow-jsonschema~=0.12.0", + "jsonschema>=4.21.1", + "marko==2.0.3", + "marshmallow-dataclass[union]~=8.6.0", + "marshmallow-jsonschema~=0.13.0", "marshmallow-union~=0.1.15", - "marshmallow~=3.13.0", + "marshmallow~=3.21.1", "pywin32 ; platform_system=='Windows'", "pytoml==0.1.21", "PyYAML~=6.0.1", - "requests~=2.27", - "toml==0.10.0", - "typing-inspect==0.8.0", - "typing-extensions==4.8.0", - "XlsxWriter~=1.3.6", - "semver==3.0.0-dev.4" + "requests~=2.31.0", + "toml==0.10.2", + "typing-inspect==0.9.0", + "typing-extensions==4.10.0", + "XlsxWriter~=3.2.0", + "semver==3.0.2" ] [project.optional-dependencies] -dev = ["pep8-naming==0.7.0", "PyGithub==1.55", "flake8==7.0.0", "pyflakes==3.2.0", "pytest>=3.6", "pre-commit==2.20.0"] +dev = ["pep8-naming==0.13.0", "PyGithub==2.2.0", "flake8==7.0.0", "pyflakes==3.2.0", "pytest>=8.1.1", "pre-commit==3.6.2"] [project.urls] "Homepage" = "https://github.com/elastic/detection-rules"