[FR] Adjust Prebuilt Rules Packaging to Use Elastic Package v3 (#3252)

* Adding support for elastic package version 3

* replaced OS with Pathlib where applicable

* added sub-dataclasses for V3

* fixed flake errors

* adjusted registry dataclasses to inherit base

Removed changes from:
- detection_rules/etc/packages.yml

(selectively cherry picked from commit cdeb398ab3)

detection_rules/devtools.py

@@ -205,7 +205,6 @@ def bump_versions(major_release: bool, minor_release: bool, patch_release: bool,
         pkg_data["name"] = f"{minor_bump.major}.{minor_bump.minor}"
         pkg_data["registry_data"]["conditions"]["kibana.version"] = f"^{pkg_kibana_ver.bump_minor()}"
         pkg_data["registry_data"]["version"] = str(pkg_ver.bump_minor().bump_prerelease("beta"))
-        pkg_data["registry_data"]["release"] = maturity
     if patch_release:
         latest_patch_release_ver = find_latest_integration_version("security_detection_engine",
                                                                    maturity, pkg_data["name"])
@@ -537,7 +536,7 @@ def kibana_pr(ctx: click.Context, label: Tuple[str, ...], assign: Tuple[str, ...
 @click.option("--token", required=True, prompt=get_github_token() is None, default=get_github_token(),
               help="GitHub token to use for the PR", hide_input=True)
 @click.option("--pkg-directory", "-d", help="Directory to save the package in cloned repository",
-              default=os.path.join("packages", "security_detection_engine"))
+              default=Path("packages", "security_detection_engine"))
 @click.option("--base-branch", "-b", help="Base branch in target repository", default="main")
 @click.option("--branch-name", "-n", help="New branch for the rules commit")
 @click.option("--github-repo", "-r", help="Repository to use for the branch", default="elastic/integrations")
@@ -556,13 +555,13 @@ def integrations_pr(ctx: click.Context, local_repo: str, token: str, draft: bool
     repo = client.get_repo(github_repo)
 
     # Use elastic-package to format and lint
-    gopath = utils.gopath()
+    gopath = utils.gopath().strip("'\"")
     assert gopath is not None, "$GOPATH isn't set"
 
     err = 'elastic-package missing, run: go install github.com/elastic/elastic-package@latest and verify go bin path'
     assert subprocess.check_output(['elastic-package'], stderr=subprocess.DEVNULL), err
 
-    local_repo = os.path.abspath(local_repo)
+    local_repo = Path(local_repo).resolve()
     stack_version = Package.load_configs()["name"]
     package_version = Package.load_configs()["registry_data"]["version"]
 
@@ -574,7 +573,7 @@ def integrations_pr(ctx: click.Context, local_repo: str, token: str, draft: bool
         click.echo(f"Run {click.style('python -m detection_rules dev build-release', bold=True)} to populate", err=True)
         ctx.exit(1)
 
-    if not Path(local_repo).exists():
+    if not local_repo.exists():
         click.secho(f"{github_repo} is not present at {local_repo}.", fg="red", err=True)
         ctx.exit(1)
 
@@ -593,7 +592,7 @@ def integrations_pr(ctx: click.Context, local_repo: str, token: str, draft: bool
     git("checkout", "-b", branch_name)
 
     # Load the changelog in memory, before it's removed. Come back for it after the PR is created
-    target_directory = Path(local_repo) / pkg_directory
+    target_directory = local_repo / pkg_directory
     changelog_path = target_directory / "changelog.yml"
     changelog_entries: list = yaml.safe_load(changelog_path.read_text(encoding="utf-8"))
 
@@ -624,13 +623,15 @@ def integrations_pr(ctx: click.Context, local_repo: str, token: str, draft: bool
 
     def elastic_pkg(*args):
         """Run a command with $GOPATH/bin/elastic-package in the package directory."""
-        prev = os.path.abspath(os.getcwd())
+        prev = Path.cwd()
         os.chdir(target_directory)
 
         try:
-            return subprocess.check_call([os.path.join(gopath, "bin", "elastic-package")] + list(args))
+            elastic_pkg_cmd = [str(Path(gopath, "bin", "elastic-package"))]
+            elastic_pkg_cmd.extend(list(args))
+            return subprocess.check_call(elastic_pkg_cmd)
         finally:
-            os.chdir(prev)
+            os.chdir(str(prev))
 
     elastic_pkg("format")