From b04218ec21df0dd516342b4a2794a149d06ebce5 Mon Sep 17 00:00:00 2001 From: Justin Ibarra Date: Wed, 17 Feb 2021 23:49:40 -0900 Subject: [PATCH] [CLI] Add repo option to kibana-diff command (#952) --- detection_rules/devtools.py | 5 +++-- detection_rules/misc.py | 22 +++++++++++----------- detection_rules/rule.py | 2 +- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/detection_rules/devtools.py b/detection_rules/devtools.py index 0edaba793..7e3691519 100644 --- a/detection_rules/devtools.py +++ b/detection_rules/devtools.py @@ -79,9 +79,10 @@ def update_lock_versions(rule_ids): @dev_group.command('kibana-diff') @click.option('--rule-id', '-r', multiple=True, help='Optionally specify rule ID') +@click.option('--repo', default='elastic/kibana', help='Repository where branch is located') @click.option('--branch', '-b', default='master', help='Specify the kibana branch to diff against') @click.option('--threads', '-t', type=click.IntRange(1), default=50, help='Number of threads to use to download rules') -def kibana_diff(rule_id, branch, threads): +def kibana_diff(rule_id, repo, branch, threads): """Diff rules against their version represented in kibana if exists.""" from .misc import get_kibana_rules @@ -94,7 +95,7 @@ def kibana_diff(rule_id, branch, threads): manage_versions(list(rules.values()), verbose=False) repo_hashes = {r.id: r.get_hash() for r in rules.values()} - kibana_rules = {r['rule_id']: r for r in get_kibana_rules(branch=branch, threads=threads).values()} + kibana_rules = {r['rule_id']: r for r in get_kibana_rules(repo=repo, branch=branch, threads=threads).values()} kibana_hashes = {r['rule_id']: Rule.dict_hash(r) for r in kibana_rules.values()} missing_from_repo = list(set(kibana_hashes).difference(set(repo_hashes))) diff --git a/detection_rules/misc.py b/detection_rules/misc.py index 35d5da54f..c78bf65e6 100644 --- a/detection_rules/misc.py +++ b/detection_rules/misc.py @@ -490,32 +490,32 @@ def schema_prompt(name, value=None, required=False, **options): return -def get_kibana_rules_map(branch='master'): +def get_kibana_rules_map(repo='elastic/kibana', branch='master'): """Get list of available rules from the Kibana repo and return a list of URLs.""" # ensure branch exists - r = requests.get(f'https://api.github.com/repos/elastic/kibana/branches/{branch}') + r = requests.get(f'https://api.github.com/repos/{repo}/branches/{branch}') r.raise_for_status() - url = ('https://api.github.com/repos/elastic/kibana/contents/x-pack/{legacy}plugins/{app}/server/lib/' + url = ('https://api.github.com/repos/{repo}/contents/x-pack/{legacy}plugins/{app}/server/lib/' 'detection_engine/rules/prepackaged_rules?ref={branch}') - gh_rules = requests.get(url.format(legacy='', app='security_solution', branch=branch)).json() + gh_rules = requests.get(url.format(legacy='', app='security_solution', branch=branch, repo=repo)).json() # pre-7.9 app was siem if isinstance(gh_rules, dict) and gh_rules.get('message', '') == 'Not Found': - gh_rules = requests.get(url.format(legacy='', app='siem', branch=branch)).json() + gh_rules = requests.get(url.format(legacy='', app='siem', branch=branch, repo=repo)).json() # pre-7.8 the siem was under the legacy directory if isinstance(gh_rules, dict) and gh_rules.get('message', '') == 'Not Found': - gh_rules = requests.get(url.format(legacy='legacy/', app='siem', branch=branch)).json() + gh_rules = requests.get(url.format(legacy='legacy/', app='siem', branch=branch, repo=repo)).json() if isinstance(gh_rules, dict) and gh_rules.get('message', '') == 'Not Found': - raise ValueError(f'rules directory does not exist for branch: {branch}') + raise ValueError(f'rules directory does not exist for {repo} branch: {branch}') return {os.path.splitext(r['name'])[0]: r['download_url'] for r in gh_rules if r['name'].endswith('.json')} -def get_kibana_rules(*rule_paths, branch='master', verbose=True, threads=50): +def get_kibana_rules(*rule_paths, repo='elastic/kibana', branch='master', verbose=True, threads=50): """Retrieve prepackaged rules from kibana repo.""" from multiprocessing.pool import ThreadPool @@ -523,11 +523,11 @@ def get_kibana_rules(*rule_paths, branch='master', verbose=True, threads=50): if verbose: thread_use = f' using {threads} threads' if threads > 1 else '' - click.echo(f'Downloading rules from {branch} branch in kibana repo{thread_use} ...') + click.echo(f'Downloading rules from {repo} {branch} branch in kibana repo{thread_use} ...') rule_paths = [os.path.splitext(os.path.basename(p))[0] for p in rule_paths] - rules_mapping = [(n, u) for n, u in get_kibana_rules_map(branch).items() if n in rule_paths] if rule_paths else \ - get_kibana_rules_map(branch).items() + rules_mapping = [(n, u) for n, u in get_kibana_rules_map(repo=repo, branch=branch).items() if n in rule_paths] \ + if rule_paths else get_kibana_rules_map(repo=repo, branch=branch).items() def download_worker(rule_info): n, u = rule_info diff --git a/detection_rules/rule.py b/detection_rules/rule.py index 841e23bf4..8e2473178 100644 --- a/detection_rules/rule.py +++ b/detection_rules/rule.py @@ -368,7 +368,7 @@ class Rule(object): """Get the version of the rule.""" from .packaging import load_versions - rules_versions = load_versions + rules_versions = load_versions() if self.id in rules_versions: version_info = rules_versions[self.id]