[CI] Add GitHub actions workflow to lock versions across branches (#1456)

* Start job to lock versions
* Update lock-versions workflow
* Call lock-multiple script
* Fix script
* Add the lock file to staging
* pass branches to the job
* Fetch all branches and tags
* Push the branch first
* Push with upstream
* Change PR params
* Remove protections machine token
* Add 7.14.0 to the lock for min_stack_version=7.14.0
* Fix branch prefix
* Add trailing newline
* Trailing newline
* Restrict to main branch

.github/workflows/lock-versions.yml

@@ -0,0 +1,73 @@
+name: lock-versions
+on:
+  workflow_dispatch:
+      inputs:
+        branches:
+          description: 'List of branches to lock versions (ordered, comma separated)'
+          required: true
+          default: '7.13,7.14,7.15'
+
+jobs:
+  pr:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Validate the source branch
+        uses: actions/github-script@v3
+        with:
+          script: |
+            if ('refs/heads/main' !== '${{github.event.ref}}') {
+              core.setFailed('Forbidden branch, expected "main"')
+            }
+
+      - name: Checkout detection-rules
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt -r requirements-dev.txt
+
+      - name: Build release package
+        run: |
+          python -m detection_rules dev build-release
+
+      - name: Set github config
+        run: |
+          git config --global user.email "72879786+protectionsmachine@users.noreply.github.com"
+          git config --global user.name "protectionsmachine"
+
+      - name: Lock the versions
+        env:
+          BRANCHES: "${{github.event.inputs.branches}}"
+        run: |
+          ./etc/lock-multiple.sh $BRANCHES
+          git add etc/version.lock.json
+
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v3
+        with:
+          assignees: '${{github.actor}}'
+          delete-branch: true
+          branch: "version-lock-"
+          commit-message: "Locked versions for releases: ${{github.event.inputs.branches}}"
+          branch-suffix: "short-commit-hash"
+          title: 'Lock versions for releases: ${{github.event.inputs.branches}}'
+          body: |
+            Lock versions for releases: ${{github.event.inputs.branches}}.
+
+            - Autogenerated from job `lock-versions: pr`.
+
+      - name: Archive production artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: release-files
+          path: |
+            releases

detection_rules/devtools.py

@@ -187,7 +187,7 @@ def update_lock_versions(rule_ids):
     if not click.confirm(f'Are you sure you want to update hashes for {len(rules)} rules without a version bump?'):
         return
 
-    changed, new, _ = manage_versions(rules, exclude_version_update=True, add_new=False, save_changes=True)
+    changed, new, _ = manage_versions(rules, exclude_version_update=True, save_changes=True)
 
     if not changed:
         click.echo('No hashes updated')

detection_rules/version_lock.py

@@ -24,7 +24,6 @@ def _convert_lock_version(stack_version: Optional[str]) -> Version:
     return max(Version(stack_version), MIN_LOCK_VERSION_DEFAULT)
 
 
-@cached
 def get_locked_version(rule_id: str, min_stack_version: Optional[str] = None) -> Optional[int]:
     rules_versions = load_versions()
 
@@ -34,7 +33,6 @@ def get_locked_version(rule_id: str, min_stack_version: Optional[str] = None) ->
         return stack_version_info['version']
 
 
-@cached
 def get_locked_hash(rule_id: str, min_stack_version: Optional[str] = None) -> Optional[str]:
     rules_versions = load_versions()
 
@@ -147,6 +145,9 @@ def manage_versions(rules: List[TOMLRule],
         save_etc_dump(current_versions, ETC_VERSION_LOCK_FILE)
         click.echo('Updated version.lock.json file')
 
+        # reset the cache
+        load_versions.clear()
+
     if newly_deprecated:
         save_etc_dump(rule_deprecations, ETC_DEPRECATED_RULES_FILE)
         click.echo('Updated deprecated_rules.json file')

etc/lock-multiple.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+set -x
+set -e
+
+CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
+
+# switch to
+for BRANCH in $(echo $@ | sed "s/,/ /g")
+do
+    echo $BRANCH
+    git checkout $BRANCH
+    git pull
+    python -m detection_rules dev build-release --update-version-lock
+done
+
+git checkout ${CURRENT_BRANCH}

etc/version.lock.json

@@ -205,6 +205,7 @@
     "version": 1
   },
   "138c5dd5-838b-446e-b1ac-c995c7f8108a": {
+    "min_stack_version": "7.14",
     "rule_name": "Rare User Logon",
     "sha256": "0f58d631c0e3181b8d45b1df57e496be523f9725dd66e00035885cdc9ec60154",
     "version": 1
@@ -550,6 +551,7 @@
     "version": 5
   },
   "3115bd2c-0baa-4df0-80ea-45e474b5ef93": {
+    "min_stack_version": "7.14",
     "rule_name": "Agent Spoofing - Mismatched Agent ID",
     "sha256": "64619f9caffb2d5207658b5ddb16c86462b4c19c8567280b74c5191166c42a25",
     "version": 1
@@ -740,6 +742,7 @@
     "version": 4
   },
   "3f0e5410-a4bf-4e8c-bcfc-79d67a285c54": {
+    "min_stack_version": "7.14",
     "rule_name": "CyberArk Privileged Access Security Error",
     "sha256": "420e91f52a8fb273a099a96a3b3e8beb4c682a608f9ce67d763b32fa803a83dd",
     "version": 1
@@ -830,6 +833,7 @@
     "version": 1
   },
   "493834ca-f861-414c-8602-150d5505b777": {
+    "min_stack_version": "7.14",
     "rule_name": "Agent Spoofing - Multiple Hosts Using Same Agent",
     "sha256": "f8e4481e5c38326daea5818415a4f06be1da64247686974940283c6b7a31f81f",
     "version": 1
@@ -1325,6 +1329,7 @@
     "version": 2
   },
   "745b0119-0560-43ba-860a-7235dd8cee8d": {
+    "min_stack_version": "7.14",
     "rule_name": "Unusual Hour for a User to Logon",
     "sha256": "cfc6d020a4aff760e43c4f33a76f8e3f56c9aca58b2199c4c498bb3f6f966b42",
     "version": 1
@@ -1710,6 +1715,7 @@
     "version": 2
   },
   "99dcf974-6587-4f65-9252-d866a3fdfd9c": {
+    "min_stack_version": "7.14",
     "rule_name": "Spike in Failed Logon Events",
     "sha256": "2638483670e005d8b56dfdea27e389782690b3216a07adb454110f0d1a27e141",
     "version": 1
@@ -2230,6 +2236,7 @@
     "version": 8
   },
   "c5f81243-56e0-47f9-b5bb-55a5ed89ba57": {
+    "min_stack_version": "7.14",
     "rule_name": "CyberArk Privileged Access Security Recommended Monitor",
     "sha256": "0c5ec551b85d7e7e8775c4c1508a831c6019881d679e137e6f0531968d3ab03c",
     "version": 1
@@ -2445,6 +2452,7 @@
     "version": 2
   },
   "d4b73fa0-9d43-465e-b8bf-50230da6718b": {
+    "min_stack_version": "7.14",
     "rule_name": "Unusual Source IP for a User to Logon from",
     "sha256": "eaec6ceda71a7d7f2ef470443ab29248249a5782241bd0d422c6c5201dff280f",
     "version": 1
@@ -2510,6 +2518,7 @@
     "version": 6
   },
   "d7d5c059-c19a-4a96-8ae3-41496ef3bcf9": {
+    "min_stack_version": "7.14",
     "rule_name": "Spike in Logon Events",
     "sha256": "f597878752cb6e91544579901584b4938249c29026da834e202622b3194aac5b",
     "version": 1
@@ -2605,6 +2614,7 @@
     "version": 6
   },
   "e26aed74-c816-40d3-a810-48d6fbd8b2fd": {
+    "min_stack_version": "7.14",
     "rule_name": "Spike in Logon Events from a Source IP",
     "sha256": "fb4afa427f0347f94517a7191fb7a7f880941fbd2bd47289ce54bcbf5bfc67c9",
     "version": 1