From 79f575b33c747e0c3c5f7293c95f3ddab611e683 Mon Sep 17 00:00:00 2001 From: Mika Ayenson Date: Wed, 15 May 2024 15:18:39 -0500 Subject: [PATCH] [FR] Normalize yml ext to yaml (#3675) --- .github/CODEOWNERS | 2 +- .github/workflows/get-target-branches.yml | 2 +- .github/workflows/manual-backport.yml | 4 ++-- .github/workflows/release-fleet.yml | 2 +- detection_rules/cli_utils.py | 2 +- detection_rules/devtools.py | 8 ++++---- detection_rules/eswrap.py | 2 +- detection_rules/etc/{packages.yml => packages.yaml} | 0 .../etc/{rule-mapping.yml => rule-mapping.yaml} | 0 detection_rules/mappings.py | 4 ++-- detection_rules/misc.py | 2 +- detection_rules/packaging.py | 4 ++-- docs/rule_insights.md | 2 +- tests/test_all_rules.py | 2 +- tests/test_mappings.py | 2 +- tests/test_packages.py | 2 +- 16 files changed, 20 insertions(+), 20 deletions(-) rename detection_rules/etc/{packages.yml => packages.yaml} (100%) rename detection_rules/etc/{rule-mapping.yml => rule-mapping.yaml} (100%) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 2f08af6b3..317a6318d 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -8,7 +8,7 @@ lib/ @brokensound77 @mikaayenson @eric-forte-elastic rta/ @brokensound77 @mikaayenson @eric-forte-elastic # skip rta-mapping to avoid the spam -detection_rules/etc/packages.yml @brokensound77 @mikaayenson @eric-forte-elastic +detection_rules/etc/packages.yaml @brokensound77 @mikaayenson @eric-forte-elastic detection_rules/etc/*.json @brokensound77 @mikaayenson @eric-forte-elastic detection_rules/etc/*.json @brokensound77 @mikaayenson @eric-forte-elastic detection_rules/etc/*/* @brokensound77 @mikaayenson @eric-forte-elastic diff --git a/.github/workflows/get-target-branches.yml b/.github/workflows/get-target-branches.yml index 9542564a0..c0a84ad90 100644 --- a/.github/workflows/get-target-branches.yml +++ b/.github/workflows/get-target-branches.yml @@ -30,4 +30,4 @@ jobs: - id: get-branch-list run: | python -m detection_rules dev utils get-branches - echo "::set-output name=matrix::$(cat ./detection_rules/etc/target-branches.yml)" + echo "::set-output name=matrix::$(cat ./detection_rules/etc/target-branches.yaml)" diff --git a/.github/workflows/manual-backport.yml b/.github/workflows/manual-backport.yml index b2e0cb906..965c64aca 100644 --- a/.github/workflows/manual-backport.yml +++ b/.github/workflows/manual-backport.yml @@ -9,7 +9,7 @@ on: description: 'Sha256 hash of the merge commit to use in backporting' required: true exceptions: - description: 'Comma seperated list of files to skip staging e.g. detection_rules/etc/packages.yml,detection_rules/attack.py)' + description: 'Comma seperated list of files to skip staging e.g. detection_rules/etc/packages.yaml,detection_rules/attack.py)' required: false jobs: @@ -60,7 +60,7 @@ jobs: env: UNSTAGED_LIST_FILE: "../unstaged-rules.txt" run: | - VERSION=$(cat detection_rules/etc/packages.yml | grep -oP "(?<=name:\s')[\d\.]+[-\w\.]*(?=')") + VERSION=$(cat detection_rules/etc/packages.yaml | grep -oP "(?<=name:\s')[\d\.]+[-\w\.]*(?=')") python -m detection_rules dev unstage-incompatible-rules --target-stack-version "$VERSION" --exception-list "${{github.event.inputs.exceptions}}" # Track which rules were unstaged diff --git a/.github/workflows/release-fleet.yml b/.github/workflows/release-fleet.yml index bb5af132f..b94e3ea7a 100644 --- a/.github/workflows/release-fleet.yml +++ b/.github/workflows/release-fleet.yml @@ -100,7 +100,7 @@ jobs: if: github.event.inputs.package_maturity == 'ga' run: | cd detection-rules - output=$(cat detection_rules/etc/packages.yml | grep -oP '(?<=\sversion: )\S+') + output=$(cat detection_rules/etc/packages.yaml | grep -oP '(?<=\sversion: )\S+') echo "pkg_version=$output" >> $GITHUB_ENV - name: Create release tag diff --git a/detection_rules/cli_utils.py b/detection_rules/cli_utils.py index 320e8dbaa..9920d416b 100644 --- a/detection_rules/cli_utils.py +++ b/detection_rules/cli_utils.py @@ -227,6 +227,6 @@ def rule_prompt(path=None, rule_type=None, required_only=True, save=True, verbos print(' - {}'.format('\n - '.join(skipped))) # rta_mappings.add_rule_to_mapping_file(rule) - # click.echo('Placeholder added to rule-mapping.yml') + # click.echo('Placeholder added to rule-mapping.yaml') return rule diff --git a/detection_rules/devtools.py b/detection_rules/devtools.py index 943ffcb17..f6277c98b 100644 --- a/detection_rules/devtools.py +++ b/detection_rules/devtools.py @@ -195,7 +195,7 @@ def build_integration_docs(ctx: click.Context, registry_version: str, pre: str, def bump_versions(major_release: bool, minor_release: bool, patch_release: bool, new_package: str, maturity: str): """Bump the versions""" - pkg_data = load_etc_dump('packages.yml')['package'] + pkg_data = load_etc_dump('packages.yaml')['package'] kibana_ver = Version.parse(pkg_data["name"], optional_minor_and_patch=True) pkg_ver = Version.parse(pkg_data["registry_data"]["version"]) pkg_kibana_ver = Version.parse(pkg_data["registry_data"]["conditions"]["kibana.version"].lstrip("^")) @@ -236,7 +236,7 @@ def bump_versions(major_release: bool, minor_release: bool, patch_release: bool, click.echo(f"Package Kibana version: {pkg_data['registry_data']['conditions']['kibana.version']}") click.echo(f"Package version: {pkg_data['registry_data']['version']}") - save_etc_dump({"package": pkg_data}, "packages.yml") + save_etc_dump({"package": pkg_data}, "packages.yaml") @dataclasses.dataclass @@ -293,7 +293,7 @@ class GitChangeEntry: def prune_staging_area(target_stack_version: str, dry_run: bool, exception_list: list): """Prune the git staging area to remove changes to incompatible rules.""" exceptions = { - "detection_rules/etc/packages.yml", + "detection_rules/etc/packages.yaml", } exceptions.update(exception_list.split(",")) @@ -1077,7 +1077,7 @@ def utils_group(): @utils_group.command('get-branches') -@click.option('--outfile', '-o', type=Path, default=get_etc_path("target-branches.yml"), help='File to save output to') +@click.option('--outfile', '-o', type=Path, default=get_etc_path("target-branches.yaml"), help='File to save output to') def get_branches(outfile: Path): branch_list = get_stack_versions(drop_patch=True) target_branches = json.dumps(branch_list[:-1]) + "\n" diff --git a/detection_rules/eswrap.py b/detection_rules/eswrap.py index b8a6d1ab6..b914ddcde 100644 --- a/detection_rules/eswrap.py +++ b/detection_rules/eswrap.py @@ -371,7 +371,7 @@ def es_group(ctx: click.Context, **kwargs): @click.option('--query', '-q', help='KQL query to scope search') @click.option('--index', '-i', multiple=True, help='Index(es) to search against (default: all indexes)') @click.option('--rta-name', '-r', help='Name of RTA in order to save events directly to unit tests data directory') -@click.option('--rule-id', help='Updates rule mapping in rule-mapping.yml file (requires --rta-name)') +@click.option('--rule-id', help='Updates rule mapping in rule-mapping.yaml file (requires --rta-name)') @click.option('--view-events', is_flag=True, help='Print events after saving') @click.pass_context def collect_events(ctx, host_id, query, index, rta_name, rule_id, view_events): diff --git a/detection_rules/etc/packages.yml b/detection_rules/etc/packages.yaml similarity index 100% rename from detection_rules/etc/packages.yml rename to detection_rules/etc/packages.yaml diff --git a/detection_rules/etc/rule-mapping.yml b/detection_rules/etc/rule-mapping.yaml similarity index 100% rename from detection_rules/etc/rule-mapping.yml rename to detection_rules/etc/rule-mapping.yaml diff --git a/detection_rules/mappings.py b/detection_rules/mappings.py index 82567dd44..ae7317ff5 100644 --- a/detection_rules/mappings.py +++ b/detection_rules/mappings.py @@ -22,7 +22,7 @@ class RtaMappings: def __init__(self): """Rta-mapping validation and prep.""" - self.mapping: dict = load_etc_dump('rule-mapping.yml') + self.mapping: dict = load_etc_dump('rule-mapping.yaml') self.validate() self._rta_mapping = defaultdict(list) @@ -48,7 +48,7 @@ class RtaMappings: mapping[rule.id] = rule_map self.mapping = dict(sorted(mapping.items())) - save_etc_dump(self.mapping, 'rule-mapping.yml') + save_etc_dump(self.mapping, 'rule-mapping.yaml') return rule_map def get_rta_mapping(self): diff --git a/detection_rules/misc.py b/detection_rules/misc.py index 26c190c6c..5c35a722d 100644 --- a/detection_rules/misc.py +++ b/detection_rules/misc.py @@ -267,7 +267,7 @@ def get_kibana_rules(*rule_paths, repo='elastic/kibana', branch='master', verbos @cached def load_current_package_version() -> str: """Load the current package version from config file.""" - return load_etc_dump('packages.yml')['package']['name'] + return load_etc_dump('packages.yaml')['package']['name'] def get_default_config() -> Optional[Path]: diff --git a/detection_rules/packaging.py b/detection_rules/packaging.py index 63114ec38..c8b9804e2 100644 --- a/detection_rules/packaging.py +++ b/detection_rules/packaging.py @@ -28,7 +28,7 @@ from .utils import Ndjson, get_path, get_etc_path, load_etc_dump from .version_lock import default_version_lock RELEASE_DIR = get_path("releases") -PACKAGE_FILE = get_etc_path('packages.yml') +PACKAGE_FILE = get_etc_path('packages.yaml') NOTICE_FILE = get_path('NOTICE.txt') FLEET_PKG_LOGO = get_etc_path("security-logo-color-64px.svg") @@ -99,7 +99,7 @@ class Package(object): @classmethod def load_configs(cls): - """Load configs from packages.yml.""" + """Load configs from packages.yaml.""" return load_etc_dump(PACKAGE_FILE)['package'] @staticmethod diff --git a/docs/rule_insights.md b/docs/rule_insights.md index 589d0a91e..e201d0e83 100644 --- a/docs/rule_insights.md +++ b/docs/rule_insights.md @@ -11,7 +11,7 @@ The simplest way to index rules from the repo into elasticsearch is to run `python -m detection-rules es index-rules` This will index an enriched version of all rules included and sent to the index `rules-repo--` -- `package-version` is the version defined in `detection_rules/etc/packages.yml` +- `package-version` is the version defined in `detection_rules/etc/packages.yaml` - `package hash` is the sha256 hash of the consolidated rules: - sorted by name - flattened diff --git a/tests/test_all_rules.py b/tests/test_all_rules.py index 97edd6f70..fa432696c 100644 --- a/tests/test_all_rules.py +++ b/tests/test_all_rules.py @@ -75,7 +75,7 @@ class TestValidRules(BaseRuleTest): def test_production_rules_have_rta(self): """Ensure that all production rules have RTAs.""" - mappings = load_etc_dump('rule-mapping.yml') + mappings = load_etc_dump('rule-mapping.yaml') ttp_names = sorted(get_available_tests()) for rule in self.production_rules: diff --git a/tests/test_mappings.py b/tests/test_mappings.py index 77a1b05ba..cd6529a9c 100644 --- a/tests/test_mappings.py +++ b/tests/test_mappings.py @@ -28,7 +28,7 @@ class TestMappings(BaseRuleTest): def test_true_positives(self): """Test that expected results return against true positives.""" mismatched_ecs = [] - mappings = load_etc_dump('rule-mapping.yml') + mappings = load_etc_dump('rule-mapping.yaml') for rule in self.production_rules: if rule.contents.data.type == "query" and rule.contents.data.language == "kuery": diff --git a/tests/test_packages.py b/tests/test_packages.py index ca014cbee..648d3cfcf 100644 --- a/tests/test_packages.py +++ b/tests/test_packages.py @@ -55,7 +55,7 @@ class TestPackages(BaseRuleTest): """Test that packages are loading correctly.""" def test_package_loader_default_configs(self): - """Test configs in detection_rules/etc/packages.yml.""" + """Test configs in detection_rules/etc/packages.yaml.""" Package.from_config(package_configs) def test_package_summary(self):