security-tools/sigma-rules
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[Bug] Adjust build-release CLI and fix links when generating security docs (#3434)

Browse Source 
* removed historical argument; added setup string; fixed links

* fixing flake errors

* added types for command arguments

* adjusted get_release_diff to append strings for release tags

* set fetch-depth to 0 for integrations checkout in workflow

* changed the name of the workflow

* removed TODOs

* adjusted release docs workflow to remove prefix for release tags

* adjusted URL replacement only if pointed to docs site

* added elastic website to regex pattern

* add docstrings; adjusted regex; add note for stopgap

* added a note about the regex pattern for elastic URLs
This commit is contained in:
Terrance DeJesus
2024-02-12 10:08:06 -05:00
committed by GitHub
parent 298d1bce0d
commit 06b97ec79b
5 changed files with 45 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/release-docs.yml
+2 -2
View File
@@ -53,8 +53,8 @@ jobs:
env:
UPDATE_MESSAGE: ${{ github.event.inputs.update_message }}
REGISTRY_VERSION: ${{ github.event.inputs.post_version }}
PRE_VERSION: "integration-v${{ github.event.inputs.pre_version }}"
POST_VERSION: "integration-v${{ github.event.inputs.post_version }}"
PRE_VERSION: ${{ github.event.inputs.pre_version }}
POST_VERSION: ${{ github.event.inputs.post_version }}
run: |
cd detection-rules
python -m detection_rules dev build-integration-docs $REGISTRY_VERSION \
.github/workflows/release-fleet.yml
+2 -1
View File
@@ -1,4 +1,4 @@
name: release-fleet
name: Release Fleet
on:
workflow_dispatch:
inputs:
@@ -71,6 +71,7 @@ jobs:
token: ${{ secrets.READ_WRITE_RELEASE_FLEET }}
repository: ${{github.event.inputs.target_repo}}
path: integrations
fetch-depth: 0
- name: Set up Python 3.8
uses: actions/setup-python@v2
detection_rules/devtools.py
+22 -19
View File
@@ -86,15 +86,13 @@ def dev_group():
@click.option('--update-version-lock', '-u', is_flag=True,
help='Save version.lock.json file with updated rule versions in the package')
@click.option('--generate-navigator', is_flag=True, help='Generate ATT&CK navigator files')
@click.option('--add-historical', type=str, required=True, default="yes",
help='Generate historical package-registry files')
@click.option('--generate-docs', is_flag=True, default=False, help='Generate markdown documentation')
@click.option('--update-message', type=str, help='Update message for new package')
def build_release(config_file, update_version_lock: bool, generate_navigator: bool, add_historical: str,
def build_release(config_file, update_version_lock: bool, generate_navigator: bool, generate_docs: str,
update_message: str, release=None, verbose=True):
"""Assemble all the rules into Kibana-ready release files."""
config = load_dump(config_file)['package']
registry_data = config['registry_data']
add_historical = True if add_historical == "yes" else False
if generate_navigator:
config['generate_navigator'] = True
@@ -105,26 +103,27 @@ def build_release(config_file, update_version_lock: bool, generate_navigator: bo
if verbose:
click.echo(f'[+] Building package {config.get("name")}')
package = Package.from_config(config, verbose=verbose, historical=add_historical)
package = Package.from_config(config, verbose=verbose)
if update_version_lock:
default_version_lock.manage_versions(package.rules, save_changes=True, verbose=verbose)
package.save(verbose=verbose)
if add_historical:
previous_pkg_version = find_latest_integration_version("security_detection_engine", "ga",
registry_data['conditions']['kibana.version'].strip("^"))
sde = SecurityDetectionEngine()
historical_rules = sde.load_integration_assets(previous_pkg_version)
historical_rules = sde.transform_legacy_assets(historical_rules)
previous_pkg_version = find_latest_integration_version("security_detection_engine", "ga",
registry_data['conditions']['kibana.version'].strip("^"))
sde = SecurityDetectionEngine()
historical_rules = sde.load_integration_assets(previous_pkg_version)
historical_rules = sde.transform_legacy_assets(historical_rules)
package.add_historical_rules(historical_rules, registry_data['version'])
click.echo(f'[+] Adding historical rules from {previous_pkg_version} package')
# NOTE: stopgap solution until security doc migration
if generate_docs:
click.echo(f'[+] Generating security docs for {registry_data["version"]} package')
docs = IntegrationSecurityDocsMDX(registry_data['version'], Path(f'releases/{config["name"]}-docs'),
True, historical_rules, package, note=update_message)
docs.generate()
click.echo(f'[+] Adding historical rules from {previous_pkg_version} package')
package.add_historical_rules(historical_rules, registry_data['version'])
if verbose:
package.get_package_hash(verbose=verbose)
click.echo(f'- {len(package.rules)} rules included')
@@ -136,14 +135,14 @@ def get_release_diff(pre: str, post: str, remote: Optional[str] = 'origin'
) -> (Dict[str, TOMLRule], Dict[str, TOMLRule], Dict[str, DeprecatedRule]):
"""Build documents from two git tags for an integration package."""
pre_rules = RuleCollection()
pre_rules.load_git_tag(pre, remote, skip_query_validation=True)
pre_rules.load_git_tag(f'integration-v{pre}', remote, skip_query_validation=True)
if pre_rules.errors:
click.echo(f'error loading {len(pre_rules.errors)} rule(s) from: {pre}, skipping:')
click.echo(' - ' + '\n - '.join([str(p) for p in pre_rules.errors]))
post_rules = RuleCollection()
post_rules.load_git_tag(post, remote, skip_query_validation=True)
post_rules.load_git_tag(f'integration-v{post}', remote, skip_query_validation=True)
if post_rules.errors:
click.echo(f'error loading {len(post_rules.errors)} rule(s) from: {post}, skipping:')
@@ -155,12 +154,12 @@ def get_release_diff(pre: str, post: str, remote: Optional[str] = 'origin'
@dev_group.command('build-integration-docs')
@click.argument('registry-version')
@click.option('--pre', required=True, help='Tag for pre-existing rules')
@click.option('--post', required=True, help='Tag for rules post updates')
@click.option('--pre', required=True, type=str, help='Tag for pre-existing rules')
@click.option('--post', required=True, type=str, help='Tag for rules post updates')
@click.option('--directory', '-d', type=Path, required=True, help='Output directory to save docs to')
@click.option('--force', '-f', is_flag=True, help='Bypass the confirmation prompt')
@click.option('--remote', '-r', default='origin', help='Override the remote from "origin"')
@click.option('--update-message', default='Rule Updates.', help='Update message for new package')
@click.option('--update-message', default='Rule Updates.', type=str, help='Update message for new package')
@click.pass_context
def build_integration_docs(ctx: click.Context, registry_version: str, pre: str, post: str,
directory: Path, force: bool, update_message: str,
@@ -170,6 +169,10 @@ def build_integration_docs(ctx: click.Context, registry_version: str, pre: str,
if not click.confirm(f'This will refresh tags and may overwrite local tags for: {pre} and {post}. Continue?'):
ctx.exit(1)
assert Version.parse(pre) < Version.parse(post), f'pre: {pre} is not less than post: {post}'
assert Version.parse(pre), f'pre: {pre} is not a valid semver'
assert Version.parse(post), f'post: {post} is not a valid semver'
rules_changes = get_release_diff(pre, post, remote)
docs = IntegrationSecurityDocs(registry_version, directory, True, *rules_changes, update_message=update_message)
package_dir = docs.generate()
detection_rules/docs.py
+18 -2
View File
@@ -531,6 +531,11 @@ class IntegrationRuleDetail:
self.package = package_str
self.rule_title = f'prebuilt-rule-{self.package}-{name_to_title(self.rule["name"])}'
# NOTE: This pattern is used to replace markdown links with asciidoc compatible links
# upstream in security-docs repo where CI checks fail if markdown links are used
self.elastic_hyperlink_pattern = \
r'\[.*?\]\(((?:https://(?:www\.)?elastic\.co|https://docs\.elastic\.co)/.*?)\)'
# set some defaults
self.rule.setdefault('max_signals', 100)
self.rule.setdefault('interval', '5m')
@@ -549,6 +554,8 @@ class IntegrationRuleDetail:
]
if 'note' in self.rule:
page.extend([self.guide_str(), ''])
if 'setup' in self.rule:
page.extend([self.setup_str(), ''])
if 'query' in self.rule:
page.extend([self.query_str(), ''])
if 'threat' in self.rule:
@@ -557,6 +564,7 @@ class IntegrationRuleDetail:
return '\n'.join(page)
def metadata_str(self) -> str:
"""Add the metadata section to the rule detail page."""
fields = {
'type': 'Rule type',
'index': 'Rule indices',
@@ -589,13 +597,21 @@ class IntegrationRuleDetail:
return '\n'.join(values)
def guide_str(self) -> str:
return f'{AsciiDoc.title(4, "Investigation guide")}\n\n\n{AsciiDoc.code(self.rule["note"], code="markdown")}'
"""Add the guide section to the rule detail page."""
guide = re.sub(self.elastic_hyperlink_pattern, r'\1', self.rule['note'])
return f'{AsciiDoc.title(4, "Investigation guide")}\n\n\n{AsciiDoc.code(guide, code="markdown")}'
def setup_str(self) -> str:
"""Add the setup section to the rule detail page."""
setup = re.sub(self.elastic_hyperlink_pattern, r'\1', self.rule['setup'])
return f'{AsciiDoc.title(4, "Setup")}\n\n\n{AsciiDoc.code(setup, code="markdown")}'
def query_str(self) -> str:
# TODO: code=sql - would require updating existing
"""Add the query section to the rule detail page."""
return f'{AsciiDoc.title(4, "Rule query")}\n\n\n{AsciiDoc.code(self.rule["query"])}'
def threat_mapping_str(self) -> str:
"""Add the threat mapping section to the rule detail page."""
values = [AsciiDoc.bold_kv('Framework', 'MITRE ATT&CK^TM^'), '']
for entry in self.rule['threat']:
detection_rules/packaging.py
+1 -1
View File
@@ -223,7 +223,7 @@ class Package(object):
return sha256
@classmethod
def from_config(cls, config: dict = None, verbose: bool = False, historical: bool = False) -> 'Package':
def from_config(cls, config: dict = None, verbose: bool = False, historical: bool = True) -> 'Package':
"""Load a rules package given a config."""
all_rules = RuleCollection.default()
config = config or {}