[FR] Adjust Prebuilt Rules Packaging to Use Elastic Package v3 (#3252)

* Adding support for elastic package version 3

* replaced OS with Pathlib where applicable

* added sub-dataclasses for V3

* fixed flake errors

* adjusted registry dataclasses to inherit base

detection_rules/devtools.py

@@ -205,7 +205,6 @@ def bump_versions(major_release: bool, minor_release: bool, patch_release: bool,
         pkg_data["name"] = f"{minor_bump.major}.{minor_bump.minor}"
         pkg_data["registry_data"]["conditions"]["kibana.version"] = f"^{pkg_kibana_ver.bump_minor()}"
         pkg_data["registry_data"]["version"] = str(pkg_ver.bump_minor().bump_prerelease("beta"))
-        pkg_data["registry_data"]["release"] = maturity
     if patch_release:
         latest_patch_release_ver = find_latest_integration_version("security_detection_engine",
                                                                    maturity, pkg_data["name"])
@@ -537,7 +536,7 @@ def kibana_pr(ctx: click.Context, label: Tuple[str, ...], assign: Tuple[str, ...
 @click.option("--token", required=True, prompt=get_github_token() is None, default=get_github_token(),
               help="GitHub token to use for the PR", hide_input=True)
 @click.option("--pkg-directory", "-d", help="Directory to save the package in cloned repository",
-              default=os.path.join("packages", "security_detection_engine"))
+              default=Path("packages", "security_detection_engine"))
 @click.option("--base-branch", "-b", help="Base branch in target repository", default="main")
 @click.option("--branch-name", "-n", help="New branch for the rules commit")
 @click.option("--github-repo", "-r", help="Repository to use for the branch", default="elastic/integrations")
@@ -556,13 +555,13 @@ def integrations_pr(ctx: click.Context, local_repo: str, token: str, draft: bool
     repo = client.get_repo(github_repo)
 
     # Use elastic-package to format and lint
-    gopath = utils.gopath()
+    gopath = utils.gopath().strip("'\"")
     assert gopath is not None, "$GOPATH isn't set"
 
     err = 'elastic-package missing, run: go install github.com/elastic/elastic-package@latest and verify go bin path'
     assert subprocess.check_output(['elastic-package'], stderr=subprocess.DEVNULL), err
 
-    local_repo = os.path.abspath(local_repo)
+    local_repo = Path(local_repo).resolve()
     stack_version = Package.load_configs()["name"]
     package_version = Package.load_configs()["registry_data"]["version"]
 
@@ -574,7 +573,7 @@ def integrations_pr(ctx: click.Context, local_repo: str, token: str, draft: bool
         click.echo(f"Run {click.style('python -m detection_rules dev build-release', bold=True)} to populate", err=True)
         ctx.exit(1)
 
-    if not Path(local_repo).exists():
+    if not local_repo.exists():
         click.secho(f"{github_repo} is not present at {local_repo}.", fg="red", err=True)
         ctx.exit(1)
 
@@ -593,7 +592,7 @@ def integrations_pr(ctx: click.Context, local_repo: str, token: str, draft: bool
     git("checkout", "-b", branch_name)
 
     # Load the changelog in memory, before it's removed. Come back for it after the PR is created
-    target_directory = Path(local_repo) / pkg_directory
+    target_directory = local_repo / pkg_directory
     changelog_path = target_directory / "changelog.yml"
     changelog_entries: list = yaml.safe_load(changelog_path.read_text(encoding="utf-8"))
 
@@ -624,13 +623,15 @@ def integrations_pr(ctx: click.Context, local_repo: str, token: str, draft: bool
 
     def elastic_pkg(*args):
         """Run a command with $GOPATH/bin/elastic-package in the package directory."""
-        prev = os.path.abspath(os.getcwd())
+        prev = Path.cwd()
         os.chdir(target_directory)
 
         try:
-            return subprocess.check_call([os.path.join(gopath, "bin", "elastic-package")] + list(args))
+            elastic_pkg_cmd = [str(Path(gopath, "bin", "elastic-package"))]
+            elastic_pkg_cmd.extend(list(args))
+            return subprocess.check_call(elastic_pkg_cmd)
         finally:
-            os.chdir(prev)
+            os.chdir(str(prev))
 
     elastic_pkg("format")
 

detection_rules/etc/packages.yml

@@ -10,17 +10,20 @@ package:
     - security
     conditions:
       kibana.version: ^8.12.0
+      elastic:
+        subscription: basic
     description: Prebuilt detection rules for Elastic Security
-    format_version: 1.0.0
+    format_version: 3.0.0
     icons:
     - size: 16x16
       src: /img/security-logo-color-64px.svg
       type: image/svg+xml
-    license: basic
+    source:
+      license: Elastic-2.0
     name: security_detection_engine
     owner:
       github: elastic/protections
-    release: ga
+      type: elastic
     title: Prebuilt Security Detection Rules
     type: integration
     version: 8.12.0-beta.0

detection_rules/packaging.py

@@ -14,6 +14,7 @@ import textwrap
 from collections import defaultdict
 from pathlib import Path
 from typing import Dict, Optional, Tuple
+from semver import Version
 
 import click
 import yaml
@@ -377,9 +378,15 @@ class Package(object):
 
     def _generate_registry_package(self, save_dir):
         """Generate the artifact for the oob package-storage."""
-        from .schemas.registry_package import RegistryPackageManifest
+        from .schemas.registry_package import (RegistryPackageManifestV1,
+                                               RegistryPackageManifestV3)
 
-        manifest = RegistryPackageManifest.from_dict(self.registry_data)
+        # 8.12.0+ we use elastic package v3
+        stack_version = Version.parse(self.name, optional_minor_and_patch=True)
+        if stack_version >= Version.parse('8.12.0'):
+            manifest = RegistryPackageManifestV3.from_dict(self.registry_data)
+        else:
+            manifest = RegistryPackageManifestV1.from_dict(self.registry_data)
 
         package_dir = Path(save_dir) / 'fleet' / manifest.version
         docs_dir = package_dir / 'docs'

detection_rules/schemas/registry_package.py

@@ -5,7 +5,7 @@
 
 """Definitions for packages destined for the registry."""
 
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from typing import Dict, List, Optional
 
 from .definitions import ConditionSemVer, SemVer
@@ -13,22 +13,54 @@ from ..mixins import MarshmallowDataclassMixin
 
 
 @dataclass
-class RegistryPackageManifest(MarshmallowDataclassMixin):
+class ConditionElastic:
+    subscription: str
+
+
+@dataclass
+class Condition:
+    kibana_version: str = field(metadata={"data_key": "kibana.version"})
+    elastic: ConditionElastic
+
+
+@dataclass
+class Icon:
+    size: str
+    src: str
+    type: str
+
+
+@dataclass
+class RegistryPackageManifestBase(MarshmallowDataclassMixin):
     """Base class for registry packages."""
 
     categories: List[str]
-    conditions: Dict[str, ConditionSemVer]
     description: str
     format_version: SemVer
-    icons: list
-    license: str
+    icons: List[Icon]
     name: str
     owner: Dict[str, str]
-    release: str
     title: str
     type: str
     version: SemVer
 
-    internal: Optional[bool] = None
-    policy_templates: Optional[list] = None
-    screenshots: Optional[list] = None
+    internal: Optional[bool]
+    policy_templates: Optional[List[str]]
+    screenshots: Optional[List[str]]
+
+
+@dataclass
+class RegistryPackageManifestV1(RegistryPackageManifestBase):
+    """Registry packages using elastic-package v1."""
+
+    conditions: Dict[str, ConditionSemVer]
+    license: str
+    release: str
+
+
+@dataclass
+class RegistryPackageManifestV3(RegistryPackageManifestBase):
+    """Registry packages using elastic-package v3."""
+
+    conditions: Condition
+    source: Dict[str, str]

tests/test_packages.py

@@ -6,10 +6,15 @@
 """Test that the packages are built correctly."""
 import unittest
 import uuid
+from semver import Version
+from marshmallow import ValidationError
 
 from detection_rules import rule_loader
+from detection_rules.schemas.registry_package import (RegistryPackageManifestV1,
+                                                      RegistryPackageManifestV3)
 from detection_rules.packaging import PACKAGE_FILE, Package
 from detection_rules.rule_loader import RuleCollection
+
 from tests.base import BaseRuleTest
 
 package_configs = Package.load_configs()
@@ -91,19 +96,20 @@ class TestRegistryPackage(unittest.TestCase):
 
     @classmethod
     def setUpClass(cls) -> None:
-        from detection_rules.schemas.registry_package import RegistryPackageManifest
 
         assert 'registry_data' in package_configs, f'Missing registry_data in {PACKAGE_FILE}'
         cls.registry_config = package_configs['registry_data']
-        RegistryPackageManifest.from_dict(cls.registry_config)
+        stack_version = Version.parse(cls.registry_config['conditions']['kibana.version'].strip("^"),
+                                      optional_minor_and_patch=True)
+        if stack_version >= Version.parse("8.12.0"):
+            RegistryPackageManifestV3.from_dict(cls.registry_config)
+        else:
+            RegistryPackageManifestV1.from_dict(cls.registry_config)
 
     def test_registry_package_config(self):
         """Test that the registry package is validating properly."""
-        from marshmallow import ValidationError
-        from detection_rules.schemas.registry_package import RegistryPackageManifest
-
         registry_config = self.registry_config.copy()
         registry_config['version'] += '7.1.1.'
 
         with self.assertRaises(ValidationError):
-            RegistryPackageManifest.from_dict(registry_config)
+            RegistryPackageManifestV1.from_dict(registry_config)