[FR] Update Python Dependency Versions (#3515)

(cherry picked from commit 5c3523954e)

.github/workflows/pythonpackage.yml

@@ -28,7 +28,7 @@ jobs:
 
     - name: Python Lint
       run: |
-        python -m flake8 tests detection_rules --ignore D203 --max-line-length 120
+        python -m flake8 tests detection_rules --ignore D203,N815 --max-line-length 120
 
     - name: Python License Check
       run: |

Makefile

@@ -41,11 +41,21 @@ license-check: $(VENV) deps
 .PHONY: lint
 lint: $(VENV) deps
 	@echo "LINTING"
-	$(PYTHON) -m flake8 tests detection_rules --ignore D203 --max-line-length 120
+	$(PYTHON) -m flake8 tests detection_rules --ignore D203,N815 --max-line-length 120
 
 .PHONY: test
 test: $(VENV) lint pytest
 
+.PHONY: test-cli
+test-cli: $(VENV)
+	@echo "Executing test_cli script..."
+	@./detection_rules/etc/test_cli.bash
+
+.PHONY: test-remote-cli
+test-cli: $(VENV)
+	@echo "Executing test_remote_cli script..."
+	@./detection_rules/etc/test_remote_cli.bash
+
 .PHONY: release
 release: deps
 	@echo "RELEASE: $(app_name)"

detection_rules/etc/test_cli.bash

@@ -0,0 +1,56 @@
+#!/bin/bash
+
+# Path to the virtual environment
+VENV_PATH="./env/detection-rules-build"
+
+# Activate the virtual environment
+source "$VENV_PATH/bin/activate"
+
+echo "Running detection-rules CLI tests..."
+
+echo "Refreshing redirect mappings in ATT&CK"
+python -m detection_rules dev attack refresh-redirect-mappings
+
+echo "Viewing rule: threat_intel_indicator_match_address.toml"
+python -m detection_rules view-rule rules/cross-platform/threat_intel_indicator_match_address.toml
+
+echo "Exporting rule by ID: 0a97b20f-4144-49ea-be32-b540ecc445de"
+python -m detection_rules export-rules --rule-id 0a97b20f-4144-49ea-be32-b540ecc445de
+
+echo "Updating rule data schemas"
+python -m detection_rules dev schemas update-rule-data
+
+echo "Validating rule: execution_github_new_event_action_for_pat.toml"
+python -m detection_rules validate-rule rules_building_block/execution_github_new_event_action_for_pat.toml
+
+echo "Checking licenses"
+python -m detection_rules dev license-check
+
+echo "Building release and updating version lock"
+python -m detection_rules dev build-release --update-version-lock
+
+echo "Refreshing ATT&CK data"
+python -m detection_rules dev attack refresh-data
+
+echo "Updating rules with latest ATT&CK data"
+python -m detection_rules dev attack update-rules
+
+echo "Getting target branches"
+python -m detection_rules dev utils get-branches
+
+echo "Showing latest compatible version for security_detection_engine with stack version 8.12.0"
+python -m detection_rules dev integrations show-latest-compatible --package endpoint --stack_version 8.12.0
+
+echo "Building limited rules for stack version 8.12"
+python -m detection_rules build-limited-rules --stack-version "8.12" --output-file "output_file.ndjson"
+
+echo "Building limited rules for stack version 8.12 with custom rules"
+python -m detection_rules generate-rules-index --overwrite
+
+echo "Building manifests for integrations"
+python -m detection_rules dev integrations build-manifests -i endpoint
+
+echo "Building schemas for integrations"
+python -m detection_rules dev integrations build-schemas -i endpoint
+
+echo "Detection-rules CLI tests completed!"

detection_rules/etc/test_remote_cli.bash

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+# Path to the virtual environment
+VENV_PATH="./env/detection-rules-build"
+
+# Activate the virtual environment
+source "$VENV_PATH/bin/activate"
+
+echo "Running detection-rules remote CLI tests..."
+
+echo "Performing a quick rule alerts search..."
+echo "Requires .detection-rules-cfg.json credentials file set."
+python -m detection_rules kibana search-alerts
+
+echo "Detection-rules CLI tests completed!"

detection_rules/misc.py

@@ -431,13 +431,16 @@ def add_client(*client_type, add_to_ctx=True, add_func_arg=True):
             if 'kibana' in client_type:
                 # for nested ctx invocation, no need to re-auth if an existing client is already passed
                 kibana_client: Kibana = kwargs.get('kibana_client')
-                try:
-                    with kibana_client:
-                        if kibana_client and isinstance(kibana_client, Kibana) and kibana_client.version:
-                            pass
-                        else:
-                            kibana_client = get_kibana_client(**kibana_client_args)
-                except (requests.HTTPError, AttributeError):
+                if kibana_client and isinstance(kibana_client, Kibana):
+
+                    try:
+                        with kibana_client:
+                            if kibana_client.version:
+                                pass  # kibana_client is valid and can be used directly
+                    except (requests.HTTPError, AttributeError):
+                        kibana_client = get_kibana_client(**kibana_client_args)
+                else:
+                    # Instantiate a new Kibana client if none was provided or if the provided one is not usable
                     kibana_client = get_kibana_client(**kibana_client_args)
 
                 if add_func_arg:

lib/kibana/pyproject.toml

@@ -16,7 +16,7 @@ classifiers = [
 requires-python = ">=3.12"
 dependencies = [
     "requests>=2.25,<3.0",
-    "elasticsearch~=8.1",
+    "elasticsearch~=8.12.1",
 ]
 
 [project.urls]

lib/kql/pyproject.toml

@@ -16,7 +16,7 @@ classifiers = [
 requires-python = ">=3.12"
 dependencies = [
     "eql==0.9.19",
-    "lark-parser>=0.11.1",
+    "lark-parser>=0.12.0",
 ]
 
 [project.urls]

pyproject.toml

@@ -19,28 +19,28 @@ classifiers = [
   "Topic :: Utilities"
 ]
 dependencies = [
-  	"Click~=8.1.0",
-	"elasticsearch~=8.1",
+  	"Click~=8.1.7",
+	"elasticsearch~=8.12.1",
 	"eql==0.9.19",
 	"jsl==0.2.4",
-	"jsonschema>=3.2.0",
-	"marko==2.0.1",
-	"marshmallow-dataclass[union]~=8.5.12",
-	"marshmallow-jsonschema~=0.12.0",
+	"jsonschema>=4.21.1",
+	"marko==2.0.3",
+	"marshmallow-dataclass[union]~=8.6.0",
+	"marshmallow-jsonschema~=0.13.0",
 	"marshmallow-union~=0.1.15",
-	"marshmallow~=3.13.0",
+	"marshmallow~=3.21.1",
 	"pywin32 ; platform_system=='Windows'",
 	"pytoml==0.1.21",
 	"PyYAML~=6.0.1",
-	"requests~=2.27",
-	"toml==0.10.0",
-	"typing-inspect==0.8.0",
-	"typing-extensions==4.8.0",
-	"XlsxWriter~=1.3.6",
-	"semver==3.0.0-dev.4"
+	"requests~=2.31.0",
+	"toml==0.10.2",
+	"typing-inspect==0.9.0",
+	"typing-extensions==4.10.0",
+	"XlsxWriter~=3.2.0",
+	"semver==3.0.2"
 ]
 [project.optional-dependencies]
-dev = ["pep8-naming==0.7.0", "PyGithub==1.55", "flake8==7.0.0", "pyflakes==3.2.0", "pytest>=3.6", "pre-commit==2.20.0"]
+dev = ["pep8-naming==0.13.0", "PyGithub==2.2.0", "flake8==7.0.0", "pyflakes==3.2.0", "pytest>=8.1.1", "pre-commit==3.6.2"]
 
 [project.urls]
 "Homepage" = "https://github.com/elastic/detection-rules"