diff --git a/.github/workflows/release-fleet.yml b/.github/workflows/release-fleet.yml new file mode 100644 index 000000000..9f3711726 --- /dev/null +++ b/.github/workflows/release-fleet.yml @@ -0,0 +1,89 @@ +name: release-fleet +on: + workflow_dispatch: + inputs: + target_repo: + description: 'Target repository to build a PR against' + required: true + default: 'elastic/integrations' + target_branch: + description: 'Target branch for PR base' + required: true + default: 'master' + draft: + description: 'Create a PR as draft (y/n)' + required: false + +jobs: + fleet-pr: + runs-on: ubuntu-latest + + steps: + - name: Validate the source branch + uses: actions/github-script@v3 + with: + script: | + if ('refs/heads/main' === '${{github.event.ref}}') { + core.setFailed('Forbidden branch') + } + + - name: Checkout elastic/integrations + uses: actions/checkout@v2 + with: + token: ${{ secrets.PROTECTIONS_MACHINE_TOKEN }} + ref: ${{github.event.inputs.target_branch}} + repository: ${{github.event.inputs.target_repo}} + path: integrations + + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + + - name: Install Python dependencies + run: | + cd detection-rules + python -m pip install --upgrade pip + pip install -r requirements.txt -r requirements-dev.txt + + - name: Build release package + run: | + cd detection-rules + python -m detection_rules dev build-release + + - name: Set github config + run: | + git config --global user.email "72879786+protectionsmachine@users.noreply.github.com" + git config --global user.name "protectionsmachine" + + - name: Setup go + uses: actions/setup-go@v2 + with: + go-version: '^1.16.0' + + - name: Build elastic-package + run: | + go get github.com/elastic/elastic-package + + - name: Create the PR to Integrations + env: + DRAFT_ARGS: "${{startsWith(github.event.inputs.draft,'y') && '--draft' || ' '}}" + TARGET_REPO: "${{github.event.inputs.target_repo}}" + TARGET_BRANCH: "${{github.event.inputs.target_branch}}" + LOCAL_REPO: "../integrations" + GITHUB_TOKEN: "${{ secrets.PROTECTIONS_MACHINE_TOKEN }}" + run: | + cd detection-rules + python -m detection_rules dev integrations-pr \ + $LOCAL_REPO \ + --github-repo $TARGET_REPO \ + --base-branch $TARGET_BRANCH \ + --assign ${{github.actor}} \ + $DRAFT_ARGS + + - name: Archive production artifacts + uses: actions/upload-artifact@v2 + with: + name: release-files + path: | + detection-rules/releases diff --git a/.github/workflows/release-kibana.yml b/.github/workflows/release-kibana.yml index 9e23036cd..7d7127524 100644 --- a/.github/workflows/release-kibana.yml +++ b/.github/workflows/release-kibana.yml @@ -32,6 +32,11 @@ jobs: repository: elastic/kibana path: kibana + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Install dependencies run: | cd detection-rules @@ -58,9 +63,9 @@ jobs: cd detection-rules python -m detection_rules dev kibana-pr --assign ${{github.actor}} $LABEL_ARGS $DRAFT_ARGS $BRANCH_ARGS - - name: Archive production artifacts for branch builds - uses: actions/upload-artifact@v2 - with: - name: release-files - path: | - detection-rules/releases \ No newline at end of file + - name: Archive production artifacts for branch builds + uses: actions/upload-artifact@v2 + with: + name: release-files + path: | + detection-rules/releases \ No newline at end of file diff --git a/detection_rules/devtools.py b/detection_rules/devtools.py index 8026348f4..c9b7ca0c0 100644 --- a/detection_rules/devtools.py +++ b/detection_rules/devtools.py @@ -13,15 +13,16 @@ import shutil import subprocess import textwrap import time +import typing from pathlib import Path from typing import Optional, Tuple, List import click -import typing +import yaml from elasticsearch import Elasticsearch from kibana.connector import Kibana -from . import rule_loader +from . import rule_loader, utils from .cli_utils import single_collection from .eswrap import CollectEvents, add_range_to_dsl from .ghwrap import GithubClient @@ -95,6 +96,7 @@ class GitChangeEntry: def revert(self, dry_run=False): """Run a git command to revert this change.""" + def git(*args): command_line = ["git"] + [str(arg) for arg in args] click.echo(subprocess.list2cmdline(command_line)) @@ -252,8 +254,6 @@ def add_git_args(f): def kibana_commit(ctx, local_repo: str, github_repo: str, ssh: bool, kibana_directory: str, base_branch: str, branch_name: Optional[str], message: Optional[str], push: bool) -> (str, str): """Prep a commit and push to Kibana.""" - git_exe = shutil.which("git") - package_name = Package.load_configs()["name"] release_dir = os.path.join(RELEASE_DIR, package_name) message = message or f"[Detection Rules] Add {package_name} rules" @@ -263,23 +263,17 @@ def kibana_commit(ctx, local_repo: str, github_repo: str, ssh: bool, kibana_dire click.echo(f"Run {click.style('python -m detection_rules dev build-release', bold=True)} to populate", err=True) ctx.exit(1) - if not git_exe: - click.secho("Unable to find git", err=True, fg="red") - ctx.exit(1) + git = utils.make_git("-C", local_repo) # Get the current hash of the repo - long_commit_hash = subprocess.check_output([git_exe, "rev-parse", "HEAD"], encoding="utf-8").strip() - short_commit_hash = subprocess.check_output([git_exe, "rev-parse", "--short", "HEAD"], encoding="utf-8").strip() + long_commit_hash = git("rev-parse", "HEAD") + short_commit_hash = git("rev-parse", "--short", "HEAD") try: - def git(*args, show_output=False): - method = subprocess.call if show_output else subprocess.check_output - return method([git_exe, "-C", local_repo] + list(args), encoding="utf-8") - if not os.path.exists(local_repo): click.echo(f"Kibana repository doesn't exist at {local_repo}. Cloning...") url = f"git@github.com:{github_repo}.git" if ssh else f"https://github.com/{github_repo}.git" - subprocess.check_call([git_exe, "clone", url, local_repo, "--depth", "1"]) + utils.make_git()("clone", url, local_repo, "--depth", "1") else: git("checkout", base_branch) @@ -324,7 +318,6 @@ def kibana_commit(ctx, local_repo: str, github_repo: str, ssh: bool, kibana_dire # Pending an official GitHub API # @click.option("--automerge", is_flag=True, help="Enable auto-merge on the PR") @add_git_args -@click.pass_context def kibana_pr(ctx: click.Context, label: Tuple[str, ...], assign: Tuple[str, ...], draft: bool, token: str, **kwargs): """Create a pull request to Kibana.""" branch_name, commit_hash = ctx.invoke(kibana_commit, push=True, **kwargs) @@ -344,7 +337,7 @@ def kibana_pr(ctx: click.Context, label: Tuple[str, ...], assign: Tuple[str, ... - [x] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/master/packages/kbn-i18n/README.md) """).strip() # noqa: E501 - pr = repo.create_pull(title, body, kwargs["base_branch"], branch_name, draft=draft) + pr = repo.create_pull(title, body, kwargs["base_branch"], branch_name, maintainer_can_modify=True, draft=draft) # labels could also be comma separated label = {lbl for cs_labels in label for lbl in cs_labels.split(",") if lbl} @@ -359,6 +352,167 @@ def kibana_pr(ctx: click.Context, label: Tuple[str, ...], assign: Tuple[str, ... click.echo(pr.html_url) +@dev_group.command("integrations-pr") +@click.argument("local-repo", type=click.Path(exists=True, file_okay=False, dir_okay=True), + default=get_path("..", "integrations")) +@click.option("--token", required=True, prompt=get_github_token() is None, default=get_github_token(), + help="GitHub token to use for the PR", hide_input=True) +@click.option("--pkg-directory", "-d", help="Directory to save the package in cloned repository", + default=os.path.join("packages", "security_detection_engine")) +@click.option("--base-branch", "-b", help="Base branch in target repository", default="master") +@click.option("--branch-name", "-n", help="New branch for the rules commit") +@click.option("--github-repo", "-r", help="Repository to use for the branch", default="elastic/integrations") +@click.option("--assign", multiple=True, help="GitHub users to assign the PR") +@click.option("--label", multiple=True, help="GitHub labels to add to the PR") +@click.option("--draft", is_flag=True, help="Open the PR as a draft") +@click.option("--remote", help="Override the remote from 'origin'", default="origin") +@click.pass_context +def integrations_pr(ctx: click.Context, local_repo: str, token: str, draft: bool, + pkg_directory: str, base_branch: str, remote: str, + branch_name: Optional[str], github_repo: str, assign: Tuple[str, ...], label: Tuple[str, ...]): + """Create a pull request to publish the Fleet package to elastic/integrations.""" + local_repo = os.path.abspath(local_repo) + stack_version = Package.load_configs()["name"] + package_version = Package.load_configs()["registry_data"]["version"] + + release_dir = Path(RELEASE_DIR) / stack_version / "fleet" / package_version + message = f"[Security Rules] Update security rules package to v{package_version}" + + if not release_dir.exists(): + click.secho("Release directory doesn't exist.", fg="red", err=True) + click.echo(f"Run {click.style('python -m detection_rules dev build-release', bold=True)} to populate", err=True) + ctx.exit(1) + + if not Path(local_repo).exists(): + click.secho(f"{github_repo} is not present at {local_repo}.", fg="red", err=True) + ctx.exit(1) + + # Get the most recent commit hash of detection-rules + detection_rules_git = utils.make_git() + long_commit_hash = detection_rules_git("rev-parse", "HEAD") + short_commit_hash = detection_rules_git("rev-parse", "--short", "HEAD") + + # refresh the local clone of the repository + git = utils.make_git("-C", local_repo) + git("checkout", base_branch) + git("pull", remote, base_branch) + + # Switch to a new branch in elastic/integrations + branch_name = branch_name or f"detection-rules/{package_version}-{short_commit_hash}" + git("checkout", "-b", branch_name) + + # Load the changelog in memory, before it's removed. Come back for it after the PR is created + target_directory = Path(local_repo) / pkg_directory + changelog_path = target_directory / "changelog.yml" + changelog_entries: list = yaml.safe_load(changelog_path.read_text(encoding="utf-8")) + + changelog_entries.insert(0, { + "version": package_version, + "changes": [ + # This will be changed later + {"description": "Release security rules update", "type": "enhancement", + "link": "https://github.com/elastic/integrations/pulls/0000"} + ] + }) + + # Remove existing assets and replace everything + shutil.rmtree(target_directory) + actual_target_directory = shutil.copytree(release_dir, target_directory) + assert Path(actual_target_directory).absolute() == Path(target_directory).absolute(), \ + f"Expected a copy to {pkg_directory}" + + # Add the changelog back + def save_changelog(): + with changelog_path.open("wt") as f: + # add a note for other maintainers of elastic/integrations to be careful with versions + f.write("# newer versions go on top\n") + f.write("# NOTE: please use pre-release versions (e.g. -dev.0) until a package is ready for production\n") + + yaml.dump(changelog_entries, f, allow_unicode=True, default_flow_style=False, indent=2) + + save_changelog() + + # Use elastic-package to format and lint + gopath = utils.gopath() + assert gopath is not None, "$GOPATH isn't set" + + def elastic_pkg(*args): + """Run a command with $GOPATH/bin/elastic-package in the package directory.""" + prev = os.path.abspath(os.getcwd()) + os.chdir(target_directory) + + try: + return subprocess.check_call([os.path.join(gopath, "bin", "elastic-package")] + list(args)) + finally: + os.chdir(prev) + + elastic_pkg("format") + elastic_pkg("lint") + + # Upload the files to a branch + git("add", pkg_directory) + git("commit", "-m", message) + git("push", "--set-upstream", remote, branch_name) + + # Create a pull request (not done yet, but we need the PR number) + client = GithubClient(token).authenticated_client + repo = client.get_repo(github_repo) + body = textwrap.dedent(f""" + ## What does this PR do? + Update the Security Rules package to version {package_version}. + Autogenerated from commit https://github.com/elastic/detection-rules/tree/{long_commit_hash} + + ## Checklist + + - [x] I have reviewed [tips for building integrations](https://github.com/elastic/integrations/blob/master/docs/tips_for_building_integrations.md) and this pull request is aligned with them. + - [ ] ~I have verified that all data streams collect metrics or logs.~ + - [x] I have added an entry to my package's `changelog.yml` file. + - [x] If I'm introducing a new feature, I have modified the Kibana version constraint in my package's `manifest.yml` file to point to the latest Elastic stack release (e.g. `^7.13.0`). + + ## Author's Checklist + - Install the most recently release security rules in the Detection Engine + - Install the package + - Confirm the update is available in Kibana. Click "Update X rules" or "Install X rules" + - Look at the changes made after the install and confirm they are consistent + + ## How to test this PR locally + - Perform the above checklist, and use `package-storage` to build EPR from source + + ## Related issues + None + + ## Screenshots + None + """) # noqa: E501 + + pr = repo.create_pull(message, body, base_branch, branch_name, maintainer_can_modify=True, draft=draft) + + # labels could also be comma separated + label = {lbl for cs_labels in label for lbl in cs_labels.split(",") if lbl} + + if label: + pr.add_to_labels(*sorted(label)) + + if assign: + pr.add_to_assignees(*assign) + + click.echo("PR created:") + click.echo(pr.html_url) + + # replace the changelog entry with the actual PR link + changelog_entries[0]["changes"][0]["link"] = pr.html_url + save_changelog() + + # format the yml file with elastic-package + elastic_pkg("format") + elastic_pkg("lint") + + # Push the updated changelog to the PR branch + git("add", pkg_directory) + git("commit", "-m", f"Add changelog entry for {package_version}") + git("push") + + @dev_group.command('license-check') @click.option('--ignore-directory', '-i', multiple=True, help='Directories to skip (relative to base)') @click.pass_context diff --git a/detection_rules/packaging.py b/detection_rules/packaging.py index ad64a5c9e..d0e1b7482 100644 --- a/detection_rules/packaging.py +++ b/detection_rules/packaging.py @@ -495,7 +495,12 @@ class Package(object): ## License Notice - """).lstrip() + textwrap.indent(notice_contents, prefix=" ") # noqa: E501 + """).lstrip() # noqa: E501 + + # notice only needs to be appended to the README for 7.13.x + # in 7.14+ there's a separate modal to display this + if self.name == "7.13": + textwrap.indent(notice_contents, prefix=" ") readme_file.write_text(readme_text) notice_file.write_text(notice_contents) diff --git a/detection_rules/utils.py b/detection_rules/utils.py index da03b24d1..9eb1de1ea 100644 --- a/detection_rules/utils.py +++ b/detection_rules/utils.py @@ -6,6 +6,7 @@ """Util functions.""" import base64 import contextlib +import distutils.spawn import functools import glob import gzip @@ -13,13 +14,16 @@ import hashlib import io import json import os +import shutil +import subprocess import time import zipfile from dataclasses import is_dataclass, astuple from datetime import datetime, date from pathlib import Path -from typing import Dict, Union +from typing import Dict, Union, Optional, Callable +import click import eql.utils from eql.utils import load_dump, stream_json_lines @@ -47,6 +51,20 @@ class DateTimeEncoder(json.JSONEncoder): marshmallow_schemas = {} +def gopath() -> Optional[str]: + """Retrieve $GOPATH.""" + env_path = os.getenv("GOPATH") + if env_path: + return env_path + + go_bin = distutils.spawn.find_executable("go") + if go_bin: + output = subprocess.check_output([go_bin, "env"], encoding="utf-8").splitlines() + for line in output: + if line.startswith("GOPATH="): + return line[len("GOPATH="):].strip('"') + + def dict_hash(obj: dict) -> str: """Hash a dictionary deterministically.""" raw_bytes = base64.b64encode(json.dumps(obj, sort_keys=True).encode('utf-8')) @@ -299,6 +317,27 @@ def format_command_options(ctx): return formatter.getvalue() +def make_git(*prefix_args) -> Optional[Callable]: + git_exe = shutil.which("git") + prefix_args = [str(arg) for arg in prefix_args] + + if not git_exe: + click.secho("Unable to find git", err=True, fg="red") + ctx = click.get_current_context(silent=True) + + if ctx is not None: + ctx.exit(1) + + return + + def git(*args, show_output=False): + method = subprocess.call if show_output else subprocess.check_output + full_args = [git_exe] + prefix_args + [str(arg) for arg in args] + return method(full_args, encoding="utf-8").rstrip() + + return git + + def add_params(*params): """Add parameters to a click command."""