From 5bf69b7967817a24eb5a7554e800d575ef278319 Mon Sep 17 00:00:00 2001 From: Mika Ayenson Date: Thu, 8 Dec 2022 15:49:49 -0500 Subject: [PATCH] Update package and install process (#1948) --- .github/workflows/backport.yml | 2 +- .github/workflows/get-target-branches.yml | 2 +- .github/workflows/lock-versions.yml | 2 +- .github/workflows/pythonpackage.yml | 2 +- .github/workflows/release-fleet.yml | 2 +- .github/workflows/release-kibana.yml | 2 +- .gitmodules | 0 .pre-commit-config.yaml | 26 ++++++++++ Makefile | 6 +-- README.md | 2 +- detection_rules/__init__.py | 2 + detection_rules/attack.py | 4 +- detection_rules/devtools.py | 60 ++++++++++++++++++++--- detection_rules/etc/__init__.py | 4 ++ detection_rules/ghwrap.py | 2 +- detection_rules/semver.py | 13 +++++ pyproject.toml | 56 ++++++++++++++++++++- requirements-dev.txt | 1 - requirements.txt | 22 --------- setup.cfg | 4 -- 20 files changed, 167 insertions(+), 47 deletions(-) create mode 100644 .gitmodules create mode 100644 .pre-commit-config.yaml create mode 100644 detection_rules/etc/__init__.py delete mode 100644 requirements-dev.txt delete mode 100644 requirements.txt delete mode 100644 setup.cfg diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index ffae9a39d..32a2dcfdd 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -93,7 +93,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -r requirements.txt -r requirements-dev.txt + pip install .[dev] - name: Prune non-${{matrix.target_branch}} rules env: diff --git a/.github/workflows/get-target-branches.yml b/.github/workflows/get-target-branches.yml index 9e03bba9c..553e04429 100644 --- a/.github/workflows/get-target-branches.yml +++ b/.github/workflows/get-target-branches.yml @@ -24,7 +24,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -r requirements.txt + pip install .[dev] - id: get-branch-list run: | diff --git a/.github/workflows/lock-versions.yml b/.github/workflows/lock-versions.yml index 6b2010bef..f01a7fe91 100644 --- a/.github/workflows/lock-versions.yml +++ b/.github/workflows/lock-versions.yml @@ -34,7 +34,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -r requirements.txt -r requirements-dev.txt + pip install .[dev] - name: Build release package run: | diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index cd63fab06..1e20e2527 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -22,7 +22,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -r requirements.txt + pip install .[dev] - name: Python Lint run: | diff --git a/.github/workflows/release-fleet.yml b/.github/workflows/release-fleet.yml index cabb1e869..fbd3efb48 100644 --- a/.github/workflows/release-fleet.yml +++ b/.github/workflows/release-fleet.yml @@ -49,7 +49,7 @@ jobs: run: | cd detection-rules python -m pip install --upgrade pip - pip install -r requirements.txt -r requirements-dev.txt + pip install .[dev] - name: Build release package run: | diff --git a/.github/workflows/release-kibana.yml b/.github/workflows/release-kibana.yml index 7d7127524..5f041ec8d 100644 --- a/.github/workflows/release-kibana.yml +++ b/.github/workflows/release-kibana.yml @@ -41,7 +41,7 @@ jobs: run: | cd detection-rules python -m pip install --upgrade pip - pip install -r requirements.txt -r requirements-dev.txt + pip install .[dev] - name: Build release package run: | diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..e69de29bb diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..58347fc9b --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,26 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: + - repo: https://github.com/PyCQA/flake8 + rev: 5.0.4 + hooks: + - id: flake8 + args: ['--ignore=D203,C901,E501,W503', '--max-line-length=120','--max-complexity=10', '--statistics'] + exclude: '^rta|^kql' + - repo: https://github.com/PyCQA/bandit + rev: 1.7.4 + hooks: + - id: bandit + args: ['-s', 'B101,B603,B404,B607'] + exclude: '^rta|^kql' + # Potential future rigor + # - repo: https://github.com/PyCQA/pylint + # rev: v2.15.6 + # hooks: + # - id: pylint + # language: system + # exclude: '^rta|^kql' + # - repo: https://github.com/PyCQA/isort + # rev: 5.10.1 + # hooks: + # - id: isort \ No newline at end of file diff --git a/Makefile b/Makefile index 426b7038e..d61e611ba 100644 --- a/Makefile +++ b/Makefile @@ -13,9 +13,9 @@ all: release $(VENV): - pip install virtualenv + pip3 install virtualenv virtualenv $(VENV) --python=python3.8 - $(PIP) install -r requirements.txt + $(PIP) install .[dev] $(PIP) install setuptools -U @@ -25,7 +25,7 @@ clean: .PHONY: deps deps: $(VENV) - $(PIP) install -r requirements.txt + $(PIP) install .[dev] .PHONY: pytest diff --git a/README.md b/README.md index 835f50e18..84a698027 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ Detection Rules contains more than just static rule files. This repository also Although rules can be added by manually creating `.toml` files, we don't recommend it. This repository also consists of a python module that aids rule creation and unit testing. Assuming you have Python 3.8+, run the below command to install the dependencies: ```console -$ pip install -r requirements.txt +$ pip3 install ".[dev]" Collecting jsl==0.2.4 Downloading jsl-0.2.4.tar.gz (21 kB) Collecting jsonschema==3.2.0 diff --git a/detection_rules/__init__.py b/detection_rules/__init__.py index 19f40f44f..810fd2865 100644 --- a/detection_rules/__init__.py +++ b/detection_rules/__init__.py @@ -4,8 +4,10 @@ # 2.0. """Detection rules.""" + import sys + assert (3, 8) <= sys.version_info < (4, 0), "Only Python 3.8+ supported" from . import ( # noqa: E402 diff --git a/detection_rules/attack.py b/detection_rules/attack.py index a87f5448e..1120df5a4 100644 --- a/detection_rules/attack.py +++ b/detection_rules/attack.py @@ -31,8 +31,10 @@ def load_techniques_redirect() -> dict: def get_attack_file_path() -> str: pattern = 'attack-v*.json.gz' attack_file = get_etc_glob_path(pattern) - if len(attack_file) != 1: + if len(attack_file) < 1: raise FileNotFoundError(f'Missing required {pattern} file') + elif len(attack_file) != 1: + raise FileExistsError(f'Multiple files found with {pattern} pattern. Only one is allowed') return attack_file[0] diff --git a/detection_rules/devtools.py b/detection_rules/devtools.py index b2a71019e..bac0fd15b 100644 --- a/detection_rules/devtools.py +++ b/detection_rules/devtools.py @@ -17,7 +17,7 @@ import typing import urllib.parse from collections import defaultdict from pathlib import Path -from typing import Dict, Optional, Tuple, List +from typing import Dict, List, Optional, Tuple import click import requests.exceptions @@ -26,22 +26,26 @@ from elasticsearch import Elasticsearch from eql.table import Table from kibana.connector import Kibana + from . import attack, rule_loader, utils -from .cli_utils import single_collection, multi_collection +from .cli_utils import multi_collection, single_collection from .docs import IntegrationSecurityDocs from .endgame import EndgameSchemaManager from .eswrap import CollectEvents, add_range_to_dsl from .ghwrap import GithubClient, update_gist +from .integrations import build_integrations_manifest from .main import root from .misc import PYTHON_LICENSE, add_client, client_error -from .packaging import PACKAGE_FILE, RELEASE_DIR, CURRENT_RELEASE_PATH, Package, current_stack_version -from .version_lock import VersionLockFile, default_version_lock -from .rule import AnyRuleData, BaseRuleData, DeprecatedRule, QueryRuleData, ThreatMapping, TOMLRule +from .packaging import (CURRENT_RELEASE_PATH, PACKAGE_FILE, RELEASE_DIR, + Package, current_stack_version) +from .rule import (AnyRuleData, BaseRuleData, DeprecatedRule, QueryRuleData, + ThreatMapping, TOMLRule) from .rule_loader import RuleCollection, production_filter from .schemas import definitions, get_stack_versions from .semver import Version -from .utils import dict_hash, get_path, get_etc_path, load_dump -from .integrations import build_integrations_manifest +from .utils import (dict_hash, get_etc_path, get_path, load_dump, save_etc_dump, + load_etc_dump) +from .version_lock import VersionLockFile, default_version_lock RULES_DIR = get_path('rules') GH_CONFIG = Path.home() / ".config" / "gh" / "hosts.yml" @@ -147,6 +151,45 @@ def build_integration_docs(ctx: click.Context, registry_version: str, pre: str, return docs +@dev_group.command("bump-versions") +@click.option("--major", is_flag=True, help="bump the major version") +@click.option("--minor", is_flag=True, help="bump the minor version") +@click.option("--patch", is_flag=True, help="bump the patch version") +@click.option("--package", is_flag=True, help="Update the package version in the packages.yml file") +@click.option("--kibana", is_flag=True, help="Update the kibana version in the packages.yml file") +@click.option("--registry", is_flag=True, help="Update the registry version in the packages.yml file") +def bump_versions(major, minor, patch, package, kibana, registry): + """Bump the versions""" + + package_data = load_etc_dump('packages.yml')['package'] + ver = package_data["name"] + new_version = Version(ver).bump(major, minor, patch) + + kibana_version = f"^{new_version}.0" if not patch else f"^{new_version}" + registry_version = f"{new_version}.0-dev.0" if not patch else f"{new_version}-dev.0" + + # print the new versions + click.echo(f"New package version: {new_version}") + click.echo(f"New registry data version: {registry_version}") + click.echo(f"New Kibana version: {kibana_version}") + + if package: + # update package version + package_data["name"] = str(new_version) + + if kibana: + # update kibana version + package_data["registry_data"]["conditions"]["kibana.version"] = kibana_version + + if registry: + # update registry version + package_data["registry_data"]["version"] = registry_version + # update packages.yml + + if package or kibana or registry: + save_etc_dump({"package": package_data}, "packages.yml") + + @dataclasses.dataclass class GitChangeEntry: status: str @@ -696,6 +739,7 @@ def package_stats(ctx, token, threads): def search_rule_prs(ctx, no_loop, query, columns, language, token, threads): """Use KQL or EQL to find matching rules from active GitHub PRs.""" from uuid import uuid4 + from .main import search_rules all_rules: Dict[Path, TOMLRule] = {} @@ -1044,7 +1088,9 @@ def rule_survey(ctx: click.Context, query, date_range, dump_file, hide_zero_coun elasticsearch_client: Elasticsearch = None, kibana_client: Kibana = None): """Survey rule counts.""" from kibana.resources import Signal + from .main import search_rules + # from .eswrap import parse_unique_field_results survey_results = [] diff --git a/detection_rules/etc/__init__.py b/detection_rules/etc/__init__.py new file mode 100644 index 000000000..72ea1f6e2 --- /dev/null +++ b/detection_rules/etc/__init__.py @@ -0,0 +1,4 @@ +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0; you may not use this file except in compliance with the Elastic License +# 2.0. diff --git a/detection_rules/ghwrap.py b/detection_rules/ghwrap.py index f6dd79a44..133bd7132 100644 --- a/detection_rules/ghwrap.py +++ b/detection_rules/ghwrap.py @@ -147,7 +147,7 @@ class GithubClient: @classmethod def assert_github(cls): if not Github: - raise ModuleNotFoundError('Missing PyGithub - try running `pip install -r requirements-dev.txt`') + raise ModuleNotFoundError('Missing PyGithub - try running `pip3 install .[dev]`') @property def authenticated_client(self) -> Github: diff --git a/detection_rules/semver.py b/detection_rules/semver.py index ab07c7ddc..3238d6909 100644 --- a/detection_rules/semver.py +++ b/detection_rules/semver.py @@ -30,6 +30,19 @@ class Version(tuple): return recovered_str + def bump(self, major: bool = False, minor: bool = False, patch: bool = False) -> 'Version': + """Increment the version.""" + versions = list(self) + if major: + versions[0] += 1 + if minor: + versions[1] += 1 + if patch and len(versions) > 2: + versions[-1] += 1 + elif patch and len(versions) == 2: + versions.append(1) + return Version(versions) + def max_versions(*versions: str) -> str: """Return the max versioned string.""" diff --git a/pyproject.toml b/pyproject.toml index fed528d4a..64a9b18b6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,57 @@ +[project] +name = "detection_rules" +version = "0.1.0" +description = "Detection Rules is the home for rules used by Elastic Security. This repository is used for the development, maintenance, testing, validation, and release of rules for Elastic Security’s Detection Engine." +readme = "README.md" +requires-python = ">=3.8" +license = {file = "LICENSE.txt"} +keywords = ["Detection Rules", "Continuous Monitoring", "Data Protection", "Elastic", "Elastic Endgame", "Endpoint Security"] +classifiers = [ + "Topic :: Software Development :: Build Tools", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python", + "Topic :: Security", + "Topic :: Software Development :: Libraries :: Python Modules", + "Topic :: Software Development :: Libraries", + "Topic :: Software Development :: Testing", + "Topic :: Software Development", + "Topic :: Utilities" +] +dependencies = [ + "Click~=8.1.0", + "elasticsearch~=8.1", + "eql==0.9.15", + "jsl==0.2.4", + "jsonschema==3.2.0", + "marko", + "marshmallow-dataclass[union]~=8.5.6", + "marshmallow-jsonschema~=0.12.0", + "marshmallow-union~=0.1.15", + "marshmallow~=3.13.0", + "pytoml", + "PyYAML~=5.3", + "requests~=2.27", + "toml==0.10.0", + "typing-inspect==0.7.1", + "XlsxWriter~=1.3.6" +] +[project.optional-dependencies] +dev = ["pep8-naming==0.7.0", "PyGithub==1.55", "flake8==3.8.1", "pyflakes==2.2.0", "pytest>=3.6", "pre-commit==2.20.0"] + +[project.urls] +"Homepage" = "https://github.com/elastic/detection-rules" +"Bug Reports" = "https://github.com/elastic/detection-rules/issues" +"Research" = "https://www.elastic.co/security-labs" +"Elastic" = "https://www.elastic.co" + +[tool.setuptools] +package-data = {"kql" = ["*.g"]} +packages = ["detection_rules", "kql", "kibana", "rta"] + [build-system] -requires = ["setuptools"] +requires = ["setuptools", "wheel", "setuptools_scm"] build-backend = "setuptools.build_meta" diff --git a/requirements-dev.txt b/requirements-dev.txt deleted file mode 100644 index c8331448d..000000000 --- a/requirements-dev.txt +++ /dev/null @@ -1 +0,0 @@ -PyGithub==1.55 diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 3b379e08a..000000000 --- a/requirements.txt +++ /dev/null @@ -1,22 +0,0 @@ -jsl==0.2.4 -marko -pytoml -toml==0.10.0 -requests~=2.27 -Click~=8.1.0 -PyYAML~=5.3 -eql==0.9.15 -elasticsearch~=8.1 -XlsxWriter~=1.3.6 -marshmallow~=3.13.0 -marshmallow-dataclass[union]~=8.5.6 -typing-inspect==0.7.1 - -# test deps -pyflakes==2.2.0 -flake8==3.8.1 -pep8-naming==0.7.0 -pytest>=3.6 -jsonschema==3.2.0 -marshmallow-jsonschema~=0.12.0 -marshmallow-union~=0.1.15 # needed for marshmallow-jsonschema diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 72835c841..000000000 --- a/setup.cfg +++ /dev/null @@ -1,4 +0,0 @@ -[metadata] -name = detection_rules -license = Elastic License v2 -license_file = LICENSE.txt