[CI] Publish to integrations from on-demand job (#1340)

* Add command to publish integrations PR
* Add workflow_dispatch job to publish package
* Get working directory dynamically
* Fix the repo settings
* Get the absolute path for local-repo
* Filter out 'main' branch
* Update the description for target_branch
* Fix workflow definition
* Move 'if' into job
* Update ref format
* Remove unnecessary E501 suppression
* Add a link to the full commit hash
* s/partial_args/prefix_args

.github/workflows/release-fleet.yml

@@ -0,0 +1,89 @@
+name: release-fleet
+on:
+  workflow_dispatch:
+    inputs:
+      target_repo:
+        description: 'Target repository to build a PR against'
+        required: true
+        default: 'elastic/integrations'
+      target_branch:
+        description: 'Target branch for PR base'
+        required: true
+        default: 'master'
+      draft:
+        description: 'Create a PR as draft (y/n)'
+        required: false
+
+jobs:
+  fleet-pr:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Validate the source branch
+        uses: actions/github-script@v3
+        with:
+          script: |
+            if ('refs/heads/main' === '${{github.event.ref}}') {
+              core.setFailed('Forbidden branch')
+            }
+
+      - name: Checkout elastic/integrations
+        uses: actions/checkout@v2
+        with:
+          token: ${{ secrets.PROTECTIONS_MACHINE_TOKEN }}
+          ref: ${{github.event.inputs.target_branch}}
+          repository: ${{github.event.inputs.target_repo}}
+          path: integrations
+
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+
+      - name: Install Python dependencies
+        run: |
+          cd detection-rules
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt -r requirements-dev.txt
+
+      - name: Build release package
+        run: |
+          cd detection-rules
+          python -m detection_rules dev build-release
+
+      - name: Set github config
+        run: |
+          git config --global user.email "72879786+protectionsmachine@users.noreply.github.com"
+          git config --global user.name "protectionsmachine"
+
+      - name: Setup go
+        uses: actions/setup-go@v2
+        with:
+          go-version: '^1.16.0'
+
+      - name: Build elastic-package
+        run: |
+          go get github.com/elastic/elastic-package
+
+      - name: Create the PR to Integrations
+        env:
+          DRAFT_ARGS: "${{startsWith(github.event.inputs.draft,'y') && '--draft' || ' '}}"
+          TARGET_REPO: "${{github.event.inputs.target_repo}}"
+          TARGET_BRANCH: "${{github.event.inputs.target_branch}}"
+          LOCAL_REPO: "../integrations"
+          GITHUB_TOKEN: "${{ secrets.PROTECTIONS_MACHINE_TOKEN }}"
+        run: |
+          cd detection-rules
+          python -m detection_rules dev integrations-pr \
+            $LOCAL_REPO                                 \
+            --github-repo $TARGET_REPO                  \
+            --base-branch $TARGET_BRANCH                \
+            --assign ${{github.actor}}                  \
+            $DRAFT_ARGS
+
+      - name: Archive production artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: release-files
+          path: |
+            detection-rules/releases

.github/workflows/release-kibana.yml

@@ -32,6 +32,11 @@ jobs:
           repository: elastic/kibana
           path: kibana
 
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+
       - name: Install dependencies
         run: |
           cd detection-rules
@@ -58,9 +63,9 @@ jobs:
           cd detection-rules
           python -m detection_rules dev kibana-pr --assign ${{github.actor}} $LABEL_ARGS $DRAFT_ARGS $BRANCH_ARGS
 
-    - name: Archive production artifacts for branch builds
-      uses: actions/upload-artifact@v2
-      with:
-        name: release-files
-        path: |
-          detection-rules/releases
+      - name: Archive production artifacts for branch builds
+        uses: actions/upload-artifact@v2
+        with:
+          name: release-files
+          path: |
+            detection-rules/releases

detection_rules/devtools.py

@@ -13,15 +13,16 @@ import shutil
 import subprocess
 import textwrap
 import time
+import typing
 from pathlib import Path
 from typing import Optional, Tuple, List
 
 import click
-import typing
+import yaml
 from elasticsearch import Elasticsearch
 
 from kibana.connector import Kibana
-from . import rule_loader
+from . import rule_loader, utils
 from .cli_utils import single_collection
 from .eswrap import CollectEvents, add_range_to_dsl
 from .ghwrap import GithubClient
@@ -95,6 +96,7 @@ class GitChangeEntry:
 
     def revert(self, dry_run=False):
         """Run a git command to revert this change."""
+
         def git(*args):
             command_line = ["git"] + [str(arg) for arg in args]
             click.echo(subprocess.list2cmdline(command_line))
@@ -252,8 +254,6 @@ def add_git_args(f):
 def kibana_commit(ctx, local_repo: str, github_repo: str, ssh: bool, kibana_directory: str, base_branch: str,
                   branch_name: Optional[str], message: Optional[str], push: bool) -> (str, str):
     """Prep a commit and push to Kibana."""
-    git_exe = shutil.which("git")
-
     package_name = Package.load_configs()["name"]
     release_dir = os.path.join(RELEASE_DIR, package_name)
     message = message or f"[Detection Rules] Add {package_name} rules"
@@ -263,23 +263,17 @@ def kibana_commit(ctx, local_repo: str, github_repo: str, ssh: bool, kibana_dire
         click.echo(f"Run {click.style('python -m detection_rules dev build-release', bold=True)} to populate", err=True)
         ctx.exit(1)
 
-    if not git_exe:
-        click.secho("Unable to find git", err=True, fg="red")
-        ctx.exit(1)
+    git = utils.make_git("-C", local_repo)
 
     # Get the current hash of the repo
-    long_commit_hash = subprocess.check_output([git_exe, "rev-parse", "HEAD"], encoding="utf-8").strip()
-    short_commit_hash = subprocess.check_output([git_exe, "rev-parse", "--short", "HEAD"], encoding="utf-8").strip()
+    long_commit_hash = git("rev-parse", "HEAD")
+    short_commit_hash = git("rev-parse", "--short", "HEAD")
 
     try:
-        def git(*args, show_output=False):
-            method = subprocess.call if show_output else subprocess.check_output
-            return method([git_exe, "-C", local_repo] + list(args), encoding="utf-8")
-
         if not os.path.exists(local_repo):
             click.echo(f"Kibana repository doesn't exist at {local_repo}. Cloning...")
             url = f"git@github.com:{github_repo}.git" if ssh else f"https://github.com/{github_repo}.git"
-            subprocess.check_call([git_exe, "clone", url, local_repo, "--depth", "1"])
+            utils.make_git()("clone", url, local_repo, "--depth", "1")
         else:
             git("checkout", base_branch)
 
@@ -324,7 +318,6 @@ def kibana_commit(ctx, local_repo: str, github_repo: str, ssh: bool, kibana_dire
 # Pending an official GitHub API
 # @click.option("--automerge", is_flag=True, help="Enable auto-merge on the PR")
 @add_git_args
-@click.pass_context
 def kibana_pr(ctx: click.Context, label: Tuple[str, ...], assign: Tuple[str, ...], draft: bool, token: str, **kwargs):
     """Create a pull request to Kibana."""
     branch_name, commit_hash = ctx.invoke(kibana_commit, push=True, **kwargs)
@@ -344,7 +337,7 @@ def kibana_pr(ctx: click.Context, label: Tuple[str, ...], assign: Tuple[str, ...
     - [x] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing),
           uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/master/packages/kbn-i18n/README.md)
     """).strip()  # noqa: E501
-    pr = repo.create_pull(title, body, kwargs["base_branch"], branch_name, draft=draft)
+    pr = repo.create_pull(title, body, kwargs["base_branch"], branch_name, maintainer_can_modify=True, draft=draft)
 
     # labels could also be comma separated
     label = {lbl for cs_labels in label for lbl in cs_labels.split(",") if lbl}
@@ -359,6 +352,167 @@ def kibana_pr(ctx: click.Context, label: Tuple[str, ...], assign: Tuple[str, ...
     click.echo(pr.html_url)
 
 
+@dev_group.command("integrations-pr")
+@click.argument("local-repo", type=click.Path(exists=True, file_okay=False, dir_okay=True),
+                default=get_path("..", "integrations"))
+@click.option("--token", required=True, prompt=get_github_token() is None, default=get_github_token(),
+              help="GitHub token to use for the PR", hide_input=True)
+@click.option("--pkg-directory", "-d", help="Directory to save the package in cloned repository",
+              default=os.path.join("packages", "security_detection_engine"))
+@click.option("--base-branch", "-b", help="Base branch in target repository", default="master")
+@click.option("--branch-name", "-n", help="New branch for the rules commit")
+@click.option("--github-repo", "-r", help="Repository to use for the branch", default="elastic/integrations")
+@click.option("--assign", multiple=True, help="GitHub users to assign the PR")
+@click.option("--label", multiple=True, help="GitHub labels to add to the PR")
+@click.option("--draft", is_flag=True, help="Open the PR as a draft")
+@click.option("--remote", help="Override the remote from 'origin'", default="origin")
+@click.pass_context
+def integrations_pr(ctx: click.Context, local_repo: str, token: str, draft: bool,
+                    pkg_directory: str, base_branch: str, remote: str,
+                    branch_name: Optional[str], github_repo: str, assign: Tuple[str, ...], label: Tuple[str, ...]):
+    """Create a pull request to publish the Fleet package to elastic/integrations."""
+    local_repo = os.path.abspath(local_repo)
+    stack_version = Package.load_configs()["name"]
+    package_version = Package.load_configs()["registry_data"]["version"]
+
+    release_dir = Path(RELEASE_DIR) / stack_version / "fleet" / package_version
+    message = f"[Security Rules] Update security rules package to v{package_version}"
+
+    if not release_dir.exists():
+        click.secho("Release directory doesn't exist.", fg="red", err=True)
+        click.echo(f"Run {click.style('python -m detection_rules dev build-release', bold=True)} to populate", err=True)
+        ctx.exit(1)
+
+    if not Path(local_repo).exists():
+        click.secho(f"{github_repo} is not present at {local_repo}.", fg="red", err=True)
+        ctx.exit(1)
+
+    # Get the most recent commit hash of detection-rules
+    detection_rules_git = utils.make_git()
+    long_commit_hash = detection_rules_git("rev-parse", "HEAD")
+    short_commit_hash = detection_rules_git("rev-parse", "--short", "HEAD")
+
+    # refresh the local clone of the repository
+    git = utils.make_git("-C", local_repo)
+    git("checkout", base_branch)
+    git("pull", remote, base_branch)
+
+    # Switch to a new branch in elastic/integrations
+    branch_name = branch_name or f"detection-rules/{package_version}-{short_commit_hash}"
+    git("checkout", "-b", branch_name)
+
+    # Load the changelog in memory, before it's removed. Come back for it after the PR is created
+    target_directory = Path(local_repo) / pkg_directory
+    changelog_path = target_directory / "changelog.yml"
+    changelog_entries: list = yaml.safe_load(changelog_path.read_text(encoding="utf-8"))
+
+    changelog_entries.insert(0, {
+        "version": package_version,
+        "changes": [
+            # This will be changed later
+            {"description": "Release security rules update", "type": "enhancement",
+             "link": "https://github.com/elastic/integrations/pulls/0000"}
+        ]
+    })
+
+    # Remove existing assets and replace everything
+    shutil.rmtree(target_directory)
+    actual_target_directory = shutil.copytree(release_dir, target_directory)
+    assert Path(actual_target_directory).absolute() == Path(target_directory).absolute(), \
+        f"Expected a copy to {pkg_directory}"
+
+    # Add the changelog back
+    def save_changelog():
+        with changelog_path.open("wt") as f:
+            # add a note for other maintainers of elastic/integrations to be careful with versions
+            f.write("# newer versions go on top\n")
+            f.write("# NOTE: please use pre-release versions (e.g. -dev.0) until a package is ready for production\n")
+
+            yaml.dump(changelog_entries, f, allow_unicode=True, default_flow_style=False, indent=2)
+
+    save_changelog()
+
+    # Use elastic-package to format and lint
+    gopath = utils.gopath()
+    assert gopath is not None, "$GOPATH isn't set"
+
+    def elastic_pkg(*args):
+        """Run a command with $GOPATH/bin/elastic-package in the package directory."""
+        prev = os.path.abspath(os.getcwd())
+        os.chdir(target_directory)
+
+        try:
+            return subprocess.check_call([os.path.join(gopath, "bin", "elastic-package")] + list(args))
+        finally:
+            os.chdir(prev)
+
+    elastic_pkg("format")
+    elastic_pkg("lint")
+
+    # Upload the files to a branch
+    git("add", pkg_directory)
+    git("commit", "-m", message)
+    git("push", "--set-upstream", remote, branch_name)
+
+    # Create a pull request (not done yet, but we need the PR number)
+    client = GithubClient(token).authenticated_client
+    repo = client.get_repo(github_repo)
+    body = textwrap.dedent(f"""
+    ## What does this PR do?
+    Update the Security Rules package to version {package_version}.
+    Autogenerated from commit  https://github.com/elastic/detection-rules/tree/{long_commit_hash}
+
+    ## Checklist
+
+    - [x] I have reviewed [tips for building integrations](https://github.com/elastic/integrations/blob/master/docs/tips_for_building_integrations.md) and this pull request is aligned with them.
+    - [ ] ~I have verified that all data streams collect metrics or logs.~
+    - [x] I have added an entry to my package's `changelog.yml` file.
+    - [x] If I'm introducing a new feature, I have modified the Kibana version constraint in my package's `manifest.yml` file to point to the latest Elastic stack release (e.g. `^7.13.0`).
+
+    ## Author's Checklist
+    - Install the most recently release security rules in the Detection Engine
+    - Install the package
+    - Confirm the update is available in Kibana. Click "Update X rules" or "Install X rules"
+    - Look at the changes made after the install and confirm they are consistent
+
+    ## How to test this PR locally
+    - Perform the above checklist, and use `package-storage` to build EPR from source
+
+    ## Related issues
+    None
+
+    ## Screenshots
+    None
+    """)  # noqa: E501
+
+    pr = repo.create_pull(message, body, base_branch, branch_name, maintainer_can_modify=True, draft=draft)
+
+    # labels could also be comma separated
+    label = {lbl for cs_labels in label for lbl in cs_labels.split(",") if lbl}
+
+    if label:
+        pr.add_to_labels(*sorted(label))
+
+    if assign:
+        pr.add_to_assignees(*assign)
+
+    click.echo("PR created:")
+    click.echo(pr.html_url)
+
+    # replace the changelog entry with the actual PR link
+    changelog_entries[0]["changes"][0]["link"] = pr.html_url
+    save_changelog()
+
+    # format the yml file with elastic-package
+    elastic_pkg("format")
+    elastic_pkg("lint")
+
+    # Push the updated changelog to the PR branch
+    git("add", pkg_directory)
+    git("commit", "-m", f"Add changelog entry for {package_version}")
+    git("push")
+
+
 @dev_group.command('license-check')
 @click.option('--ignore-directory', '-i', multiple=True, help='Directories to skip (relative to base)')
 @click.pass_context

detection_rules/packaging.py

@@ -495,7 +495,12 @@ class Package(object):
 
         ## License Notice
 
-        """).lstrip() + textwrap.indent(notice_contents, prefix="    ")  # noqa: E501
+        """).lstrip()  # noqa: E501
+
+        # notice only needs to be appended to the README for 7.13.x
+        # in 7.14+ there's a separate modal to display this
+        if self.name == "7.13":
+            textwrap.indent(notice_contents, prefix="    ")
 
         readme_file.write_text(readme_text)
         notice_file.write_text(notice_contents)

detection_rules/utils.py

@@ -6,6 +6,7 @@
 """Util functions."""
 import base64
 import contextlib
+import distutils.spawn
 import functools
 import glob
 import gzip
@@ -13,13 +14,16 @@ import hashlib
 import io
 import json
 import os
+import shutil
+import subprocess
 import time
 import zipfile
 from dataclasses import is_dataclass, astuple
 from datetime import datetime, date
 from pathlib import Path
-from typing import Dict, Union
+from typing import Dict, Union, Optional, Callable
 
+import click
 import eql.utils
 from eql.utils import load_dump, stream_json_lines
 
@@ -47,6 +51,20 @@ class DateTimeEncoder(json.JSONEncoder):
 marshmallow_schemas = {}
 
 
+def gopath() -> Optional[str]:
+    """Retrieve $GOPATH."""
+    env_path = os.getenv("GOPATH")
+    if env_path:
+        return env_path
+
+    go_bin = distutils.spawn.find_executable("go")
+    if go_bin:
+        output = subprocess.check_output([go_bin, "env"], encoding="utf-8").splitlines()
+        for line in output:
+            if line.startswith("GOPATH="):
+                return line[len("GOPATH="):].strip('"')
+
+
 def dict_hash(obj: dict) -> str:
     """Hash a dictionary deterministically."""
     raw_bytes = base64.b64encode(json.dumps(obj, sort_keys=True).encode('utf-8'))
@@ -299,6 +317,27 @@ def format_command_options(ctx):
     return formatter.getvalue()
 
 
+def make_git(*prefix_args) -> Optional[Callable]:
+    git_exe = shutil.which("git")
+    prefix_args = [str(arg) for arg in prefix_args]
+
+    if not git_exe:
+        click.secho("Unable to find git", err=True, fg="red")
+        ctx = click.get_current_context(silent=True)
+
+        if ctx is not None:
+            ctx.exit(1)
+
+        return
+
+    def git(*args, show_output=False):
+        method = subprocess.call if show_output else subprocess.check_output
+        full_args = [git_exe] + prefix_args + [str(arg) for arg in args]
+        return method(full_args, encoding="utf-8").rstrip()
+
+    return git
+
+
 def add_params(*params):
     """Add parameters to a click command."""