diff --git a/detection_rules/devtools.py b/detection_rules/devtools.py index f32d6af8c..ba3f2c63c 100644 --- a/detection_rules/devtools.py +++ b/detection_rules/devtools.py @@ -205,7 +205,6 @@ def bump_versions(major_release: bool, minor_release: bool, patch_release: bool, pkg_data["name"] = f"{minor_bump.major}.{minor_bump.minor}" pkg_data["registry_data"]["conditions"]["kibana.version"] = f"^{pkg_kibana_ver.bump_minor()}" pkg_data["registry_data"]["version"] = str(pkg_ver.bump_minor().bump_prerelease("beta")) - pkg_data["registry_data"]["release"] = maturity if patch_release: latest_patch_release_ver = find_latest_integration_version("security_detection_engine", maturity, pkg_data["name"]) @@ -537,7 +536,7 @@ def kibana_pr(ctx: click.Context, label: Tuple[str, ...], assign: Tuple[str, ... @click.option("--token", required=True, prompt=get_github_token() is None, default=get_github_token(), help="GitHub token to use for the PR", hide_input=True) @click.option("--pkg-directory", "-d", help="Directory to save the package in cloned repository", - default=os.path.join("packages", "security_detection_engine")) + default=Path("packages", "security_detection_engine")) @click.option("--base-branch", "-b", help="Base branch in target repository", default="main") @click.option("--branch-name", "-n", help="New branch for the rules commit") @click.option("--github-repo", "-r", help="Repository to use for the branch", default="elastic/integrations") @@ -556,13 +555,13 @@ def integrations_pr(ctx: click.Context, local_repo: str, token: str, draft: bool repo = client.get_repo(github_repo) # Use elastic-package to format and lint - gopath = utils.gopath() + gopath = utils.gopath().strip("'\"") assert gopath is not None, "$GOPATH isn't set" err = 'elastic-package missing, run: go install github.com/elastic/elastic-package@latest and verify go bin path' assert subprocess.check_output(['elastic-package'], stderr=subprocess.DEVNULL), err - local_repo = os.path.abspath(local_repo) + local_repo = Path(local_repo).resolve() stack_version = Package.load_configs()["name"] package_version = Package.load_configs()["registry_data"]["version"] @@ -574,7 +573,7 @@ def integrations_pr(ctx: click.Context, local_repo: str, token: str, draft: bool click.echo(f"Run {click.style('python -m detection_rules dev build-release', bold=True)} to populate", err=True) ctx.exit(1) - if not Path(local_repo).exists(): + if not local_repo.exists(): click.secho(f"{github_repo} is not present at {local_repo}.", fg="red", err=True) ctx.exit(1) @@ -593,7 +592,7 @@ def integrations_pr(ctx: click.Context, local_repo: str, token: str, draft: bool git("checkout", "-b", branch_name) # Load the changelog in memory, before it's removed. Come back for it after the PR is created - target_directory = Path(local_repo) / pkg_directory + target_directory = local_repo / pkg_directory changelog_path = target_directory / "changelog.yml" changelog_entries: list = yaml.safe_load(changelog_path.read_text(encoding="utf-8")) @@ -624,13 +623,15 @@ def integrations_pr(ctx: click.Context, local_repo: str, token: str, draft: bool def elastic_pkg(*args): """Run a command with $GOPATH/bin/elastic-package in the package directory.""" - prev = os.path.abspath(os.getcwd()) + prev = Path.cwd() os.chdir(target_directory) try: - return subprocess.check_call([os.path.join(gopath, "bin", "elastic-package")] + list(args)) + elastic_pkg_cmd = [str(Path(gopath, "bin", "elastic-package"))] + elastic_pkg_cmd.extend(list(args)) + return subprocess.check_call(elastic_pkg_cmd) finally: - os.chdir(prev) + os.chdir(str(prev)) elastic_pkg("format") diff --git a/detection_rules/etc/packages.yml b/detection_rules/etc/packages.yml index 199a93a8e..c8050f5c0 100644 --- a/detection_rules/etc/packages.yml +++ b/detection_rules/etc/packages.yml @@ -10,17 +10,20 @@ package: - security conditions: kibana.version: ^8.12.0 + elastic: + subscription: basic description: Prebuilt detection rules for Elastic Security - format_version: 1.0.0 + format_version: 3.0.0 icons: - size: 16x16 src: /img/security-logo-color-64px.svg type: image/svg+xml - license: basic + source: + license: Elastic-2.0 name: security_detection_engine owner: github: elastic/protections - release: ga + type: elastic title: Prebuilt Security Detection Rules type: integration version: 8.12.0-beta.0 diff --git a/detection_rules/packaging.py b/detection_rules/packaging.py index 4423c63a2..6251c68b1 100644 --- a/detection_rules/packaging.py +++ b/detection_rules/packaging.py @@ -14,6 +14,7 @@ import textwrap from collections import defaultdict from pathlib import Path from typing import Dict, Optional, Tuple +from semver import Version import click import yaml @@ -377,9 +378,15 @@ class Package(object): def _generate_registry_package(self, save_dir): """Generate the artifact for the oob package-storage.""" - from .schemas.registry_package import RegistryPackageManifest + from .schemas.registry_package import (RegistryPackageManifestV1, + RegistryPackageManifestV3) - manifest = RegistryPackageManifest.from_dict(self.registry_data) + # 8.12.0+ we use elastic package v3 + stack_version = Version.parse(self.name, optional_minor_and_patch=True) + if stack_version >= Version.parse('8.12.0'): + manifest = RegistryPackageManifestV3.from_dict(self.registry_data) + else: + manifest = RegistryPackageManifestV1.from_dict(self.registry_data) package_dir = Path(save_dir) / 'fleet' / manifest.version docs_dir = package_dir / 'docs' diff --git a/detection_rules/schemas/registry_package.py b/detection_rules/schemas/registry_package.py index b94cf13c8..7c1719c40 100644 --- a/detection_rules/schemas/registry_package.py +++ b/detection_rules/schemas/registry_package.py @@ -5,7 +5,7 @@ """Definitions for packages destined for the registry.""" -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Dict, List, Optional from .definitions import ConditionSemVer, SemVer @@ -13,22 +13,54 @@ from ..mixins import MarshmallowDataclassMixin @dataclass -class RegistryPackageManifest(MarshmallowDataclassMixin): +class ConditionElastic: + subscription: str + + +@dataclass +class Condition: + kibana_version: str = field(metadata={"data_key": "kibana.version"}) + elastic: ConditionElastic + + +@dataclass +class Icon: + size: str + src: str + type: str + + +@dataclass +class RegistryPackageManifestBase(MarshmallowDataclassMixin): """Base class for registry packages.""" categories: List[str] - conditions: Dict[str, ConditionSemVer] description: str format_version: SemVer - icons: list - license: str + icons: List[Icon] name: str owner: Dict[str, str] - release: str title: str type: str version: SemVer - internal: Optional[bool] = None - policy_templates: Optional[list] = None - screenshots: Optional[list] = None + internal: Optional[bool] + policy_templates: Optional[List[str]] + screenshots: Optional[List[str]] + + +@dataclass +class RegistryPackageManifestV1(RegistryPackageManifestBase): + """Registry packages using elastic-package v1.""" + + conditions: Dict[str, ConditionSemVer] + license: str + release: str + + +@dataclass +class RegistryPackageManifestV3(RegistryPackageManifestBase): + """Registry packages using elastic-package v3.""" + + conditions: Condition + source: Dict[str, str] diff --git a/tests/test_packages.py b/tests/test_packages.py index 25ea95626..ca014cbee 100644 --- a/tests/test_packages.py +++ b/tests/test_packages.py @@ -6,10 +6,15 @@ """Test that the packages are built correctly.""" import unittest import uuid +from semver import Version +from marshmallow import ValidationError from detection_rules import rule_loader +from detection_rules.schemas.registry_package import (RegistryPackageManifestV1, + RegistryPackageManifestV3) from detection_rules.packaging import PACKAGE_FILE, Package from detection_rules.rule_loader import RuleCollection + from tests.base import BaseRuleTest package_configs = Package.load_configs() @@ -91,19 +96,20 @@ class TestRegistryPackage(unittest.TestCase): @classmethod def setUpClass(cls) -> None: - from detection_rules.schemas.registry_package import RegistryPackageManifest assert 'registry_data' in package_configs, f'Missing registry_data in {PACKAGE_FILE}' cls.registry_config = package_configs['registry_data'] - RegistryPackageManifest.from_dict(cls.registry_config) + stack_version = Version.parse(cls.registry_config['conditions']['kibana.version'].strip("^"), + optional_minor_and_patch=True) + if stack_version >= Version.parse("8.12.0"): + RegistryPackageManifestV3.from_dict(cls.registry_config) + else: + RegistryPackageManifestV1.from_dict(cls.registry_config) def test_registry_package_config(self): """Test that the registry package is validating properly.""" - from marshmallow import ValidationError - from detection_rules.schemas.registry_package import RegistryPackageManifest - registry_config = self.registry_config.copy() registry_config['version'] += '7.1.1.' with self.assertRaises(ValidationError): - RegistryPackageManifest.from_dict(registry_config) + RegistryPackageManifestV1.from_dict(registry_config)