Update package and install process (#1948)

.github/workflows/backport.yml

@@ -93,7 +93,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install -r requirements.txt -r requirements-dev.txt
+          pip install .[dev]
 
       - name: Prune non-${{matrix.target_branch}} rules
         env:

.github/workflows/get-target-branches.yml

@@ -24,7 +24,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install -r requirements.txt
+          pip install .[dev]
 
       - id: get-branch-list
         run: |

.github/workflows/lock-versions.yml

@@ -34,7 +34,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install -r requirements.txt -r requirements-dev.txt
+          pip install .[dev]
 
       - name: Build release package
         run: |

.github/workflows/pythonpackage.yml

@@ -22,7 +22,7 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install -r requirements.txt
+        pip install .[dev]
 
     - name: Python Lint
       run: |

.github/workflows/release-fleet.yml

@@ -49,7 +49,7 @@ jobs:
         run: |
           cd detection-rules
           python -m pip install --upgrade pip
-          pip install -r requirements.txt -r requirements-dev.txt
+          pip install .[dev]
 
       - name: Build release package
         run: |

.github/workflows/release-kibana.yml

@@ -41,7 +41,7 @@ jobs:
         run: |
           cd detection-rules
           python -m pip install --upgrade pip
-          pip install -r requirements.txt -r requirements-dev.txt
+          pip install .[dev]
 
       - name: Build release package
         run: |

.pre-commit-config.yaml

@@ -0,0 +1,26 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+repos:
+  - repo: https://github.com/PyCQA/flake8
+    rev: 5.0.4
+    hooks:
+      - id: flake8
+        args: ['--ignore=D203,C901,E501,W503', '--max-line-length=120','--max-complexity=10', '--statistics']
+        exclude: '^rta|^kql'
+  - repo: https://github.com/PyCQA/bandit
+    rev: 1.7.4
+    hooks:
+      - id: bandit
+        args: ['-s', 'B101,B603,B404,B607']
+        exclude: '^rta|^kql'
+  # Potential future rigor
+  # - repo: https://github.com/PyCQA/pylint
+  #   rev: v2.15.6
+  #   hooks:
+  #     - id: pylint
+  #       language: system
+  #       exclude: '^rta|^kql'
+  # - repo: https://github.com/PyCQA/isort
+  #   rev: 5.10.1
+  #   hooks:
+  #     - id: isort

Makefile

@@ -13,9 +13,9 @@ all: release
 
 
 $(VENV):
-	pip install virtualenv
+	pip3 install virtualenv
 	virtualenv $(VENV) --python=python3.8
-	$(PIP) install -r requirements.txt
+	$(PIP) install .[dev]
 	$(PIP) install setuptools -U
 
 
@@ -25,7 +25,7 @@ clean:
 
 .PHONY: deps
 deps: $(VENV)
-	$(PIP) install -r requirements.txt
+	$(PIP) install .[dev]
 
 
 .PHONY: pytest

README.md

@@ -39,7 +39,7 @@ Detection Rules contains more than just static rule files. This repository also
 
 Although rules can be added by manually creating `.toml` files, we don't recommend it. This repository also consists of a python module that aids rule creation and unit testing. Assuming you have Python 3.8+, run the below command to install the dependencies:
 ```console
-$ pip install -r requirements.txt
+$ pip3 install ".[dev]"
 Collecting jsl==0.2.4
   Downloading jsl-0.2.4.tar.gz (21 kB)
 Collecting jsonschema==3.2.0

detection_rules/__init__.py

@@ -4,8 +4,10 @@
 # 2.0.
 
 """Detection rules."""
+
 import sys
 
+
 assert (3, 8) <= sys.version_info < (4, 0), "Only Python 3.8+ supported"
 
 from . import (  # noqa: E402

detection_rules/attack.py

@@ -31,8 +31,10 @@ def load_techniques_redirect() -> dict:
 def get_attack_file_path() -> str:
     pattern = 'attack-v*.json.gz'
     attack_file = get_etc_glob_path(pattern)
-    if len(attack_file) != 1:
+    if len(attack_file) < 1:
         raise FileNotFoundError(f'Missing required {pattern} file')
+    elif len(attack_file) != 1:
+        raise FileExistsError(f'Multiple files found with {pattern} pattern. Only one is allowed')
     return attack_file[0]
 
 

detection_rules/devtools.py

@@ -17,7 +17,7 @@ import typing
 import urllib.parse
 from collections import defaultdict
 from pathlib import Path
-from typing import Dict, Optional, Tuple, List
+from typing import Dict, List, Optional, Tuple
 
 import click
 import requests.exceptions
@@ -26,22 +26,26 @@ from elasticsearch import Elasticsearch
 from eql.table import Table
 
 from kibana.connector import Kibana
+
 from . import attack, rule_loader, utils
-from .cli_utils import single_collection, multi_collection
+from .cli_utils import multi_collection, single_collection
 from .docs import IntegrationSecurityDocs
 from .endgame import EndgameSchemaManager
 from .eswrap import CollectEvents, add_range_to_dsl
 from .ghwrap import GithubClient, update_gist
+from .integrations import build_integrations_manifest
 from .main import root
 from .misc import PYTHON_LICENSE, add_client, client_error
-from .packaging import PACKAGE_FILE, RELEASE_DIR, CURRENT_RELEASE_PATH, Package, current_stack_version
-from .version_lock import VersionLockFile, default_version_lock
-from .rule import AnyRuleData, BaseRuleData, DeprecatedRule, QueryRuleData, ThreatMapping, TOMLRule
+from .packaging import (CURRENT_RELEASE_PATH, PACKAGE_FILE, RELEASE_DIR,
+                        Package, current_stack_version)
+from .rule import (AnyRuleData, BaseRuleData, DeprecatedRule, QueryRuleData,
+                   ThreatMapping, TOMLRule)
 from .rule_loader import RuleCollection, production_filter
 from .schemas import definitions, get_stack_versions
 from .semver import Version
-from .utils import dict_hash, get_path, get_etc_path, load_dump
-from .integrations import build_integrations_manifest
+from .utils import (dict_hash, get_etc_path, get_path, load_dump, save_etc_dump,
+                    load_etc_dump)
+from .version_lock import VersionLockFile, default_version_lock
 
 RULES_DIR = get_path('rules')
 GH_CONFIG = Path.home() / ".config" / "gh" / "hosts.yml"
@@ -147,6 +151,45 @@ def build_integration_docs(ctx: click.Context, registry_version: str, pre: str,
     return docs
 
 
+@dev_group.command("bump-versions")
+@click.option("--major", is_flag=True, help="bump the major version")
+@click.option("--minor", is_flag=True, help="bump the minor version")
+@click.option("--patch", is_flag=True, help="bump the patch version")
+@click.option("--package", is_flag=True, help="Update the package version in the packages.yml file")
+@click.option("--kibana", is_flag=True, help="Update the kibana version in the packages.yml file")
+@click.option("--registry", is_flag=True, help="Update the registry version in the packages.yml file")
+def bump_versions(major, minor, patch, package, kibana, registry):
+    """Bump the versions"""
+
+    package_data = load_etc_dump('packages.yml')['package']
+    ver = package_data["name"]
+    new_version = Version(ver).bump(major, minor, patch)
+
+    kibana_version = f"^{new_version}.0" if not patch else f"^{new_version}"
+    registry_version = f"{new_version}.0-dev.0" if not patch else f"{new_version}-dev.0"
+
+    # print the new versions
+    click.echo(f"New package version: {new_version}")
+    click.echo(f"New registry data version: {registry_version}")
+    click.echo(f"New Kibana version: {kibana_version}")
+
+    if package:
+        # update package version
+        package_data["name"] = str(new_version)
+
+    if kibana:
+        # update kibana version
+        package_data["registry_data"]["conditions"]["kibana.version"] = kibana_version
+
+    if registry:
+        # update registry version
+        package_data["registry_data"]["version"] = registry_version
+        # update packages.yml
+
+    if package or kibana or registry:
+        save_etc_dump({"package": package_data}, "packages.yml")
+
+
 @dataclasses.dataclass
 class GitChangeEntry:
     status: str
@@ -696,6 +739,7 @@ def package_stats(ctx, token, threads):
 def search_rule_prs(ctx, no_loop, query, columns, language, token, threads):
     """Use KQL or EQL to find matching rules from active GitHub PRs."""
     from uuid import uuid4
+
     from .main import search_rules
 
     all_rules: Dict[Path, TOMLRule] = {}
@@ -1044,7 +1088,9 @@ def rule_survey(ctx: click.Context, query, date_range, dump_file, hide_zero_coun
                 elasticsearch_client: Elasticsearch = None, kibana_client: Kibana = None):
     """Survey rule counts."""
     from kibana.resources import Signal
+
     from .main import search_rules
+
     # from .eswrap import parse_unique_field_results
 
     survey_results = []

detection_rules/etc/__init__.py

@@ -0,0 +1,4 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License
+# 2.0; you may not use this file except in compliance with the Elastic License
+# 2.0.

detection_rules/ghwrap.py

@@ -147,7 +147,7 @@ class GithubClient:
     @classmethod
     def assert_github(cls):
         if not Github:
-            raise ModuleNotFoundError('Missing PyGithub - try running `pip install -r requirements-dev.txt`')
+            raise ModuleNotFoundError('Missing PyGithub - try running `pip3 install .[dev]`')
 
     @property
     def authenticated_client(self) -> Github:

detection_rules/semver.py

@@ -30,6 +30,19 @@ class Version(tuple):
 
         return recovered_str
 
+    def bump(self, major: bool = False, minor: bool = False, patch: bool = False) -> 'Version':
+        """Increment the version."""
+        versions = list(self)
+        if major:
+            versions[0] += 1
+        if minor:
+            versions[1] += 1
+        if patch and len(versions) > 2:
+            versions[-1] += 1
+        elif patch and len(versions) == 2:
+            versions.append(1)
+        return Version(versions)
+
 
 def max_versions(*versions: str) -> str:
     """Return the max versioned string."""

pyproject.toml

@@ -1,3 +1,57 @@
+[project]
+name = "detection_rules"
+version = "0.1.0"
+description = "Detection Rules is the home for rules used by Elastic Security. This repository is used for the development, maintenance, testing, validation, and release of rules for Elastic Security’s Detection Engine."
+readme = "README.md"
+requires-python = ">=3.8"
+license = {file = "LICENSE.txt"}
+keywords = ["Detection Rules", "Continuous Monitoring", "Data Protection", "Elastic", "Elastic Endgame", "Endpoint Security"]
+classifiers = [
+  "Topic :: Software Development :: Build Tools",
+  "Operating System :: OS Independent",
+  "Programming Language :: Python :: 3.8",
+  "Programming Language :: Python :: 3.9",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python",
+  "Topic :: Security",
+  "Topic :: Software Development :: Libraries :: Python Modules",
+  "Topic :: Software Development :: Libraries",
+  "Topic :: Software Development :: Testing",
+  "Topic :: Software Development",
+  "Topic :: Utilities"
+]
+dependencies = [
+  	"Click~=8.1.0",
+	"elasticsearch~=8.1",
+	"eql==0.9.15",
+	"jsl==0.2.4",
+	"jsonschema==3.2.0",
+	"marko",
+	"marshmallow-dataclass[union]~=8.5.6",
+	"marshmallow-jsonschema~=0.12.0",
+	"marshmallow-union~=0.1.15",
+	"marshmallow~=3.13.0",
+	"pytoml",
+	"PyYAML~=5.3",
+	"requests~=2.27",
+	"toml==0.10.0",
+	"typing-inspect==0.7.1",
+	"XlsxWriter~=1.3.6"
+]
+[project.optional-dependencies]
+dev = ["pep8-naming==0.7.0", "PyGithub==1.55", "flake8==3.8.1", "pyflakes==2.2.0", "pytest>=3.6", "pre-commit==2.20.0"]
+
+[project.urls]
+"Homepage" = "https://github.com/elastic/detection-rules"
+"Bug Reports" = "https://github.com/elastic/detection-rules/issues"
+"Research" = "https://www.elastic.co/security-labs"
+"Elastic" = "https://www.elastic.co"
+
+[tool.setuptools]
+package-data = {"kql" = ["*.g"]}
+packages = ["detection_rules",	"kql",	"kibana",	"rta"]
+
 [build-system]
-requires = ["setuptools"]
+requires = ["setuptools", "wheel", "setuptools_scm"]
 build-backend = "setuptools.build_meta"

requirements-dev.txt

@@ -1 +0,0 @@
-PyGithub==1.55

requirements.txt

@@ -1,22 +0,0 @@
-jsl==0.2.4
-marko
-pytoml
-toml==0.10.0
-requests~=2.27
-Click~=8.1.0
-PyYAML~=5.3
-eql==0.9.15
-elasticsearch~=8.1
-XlsxWriter~=1.3.6
-marshmallow~=3.13.0
-marshmallow-dataclass[union]~=8.5.6
-typing-inspect==0.7.1
-
-# test deps
-pyflakes==2.2.0
-flake8==3.8.1
-pep8-naming==0.7.0
-pytest>=3.6
-jsonschema==3.2.0
-marshmallow-jsonschema~=0.12.0
-marshmallow-union~=0.1.15  # needed for marshmallow-jsonschema

setup.cfg

@@ -1,4 +0,0 @@
-[metadata]
-name = detection_rules
-license = Elastic License v2
-license_file = LICENSE.txt