[Bug] Update Prebuilt Detection Rules Release Process (#3403)

* release fleet workflow updates; build package integration reference changes

* updated commit hash extraction to output to env

* adjusted bump-pkg-versions to only include release if necessary

* fixed flake errors

* add historical argument for build-release set to yes by default

* Update detection_rules/devtools.py

* fixed fleet workflow; updated registry data references

* updated job names

* removed extract commit hash job and consolidated into fleet pr job

* added echo statement for current branch before checkout

* removed id from extract commit hash

.github/workflows/release-fleet.yml

@@ -28,162 +28,128 @@ on:
         type: choice
         description: 'New Package'
         required: true
+        default: "true"
         options:
           - "true"
           - "false"
-      add_historical:
-        type: choice
-        description: 'Add Historical Rules'
-        required: true
-        options:
-          - "yes"
-          - "no"
-      commit_hash:
-        description: 'Commit hash'
-        required: true
 
 jobs:
-  check-commit:
-    name: Check Commit Hash
-    runs-on: ubuntu-latest
-    outputs:
-      is_locked_commit: ${{ steps.check_commit.outputs.check_message }}
-    steps:
-      - name: Checkout detection-rules
-        uses: actions/checkout@v3
-        with:
-          path: detection-rules
-          fetch-depth: 0
+    fleet-pr:
+      name: Build package and create PR to integrations
+      runs-on: ubuntu-latest
+      steps:
+        - name: Validate the source branch
+          uses: actions/github-script@v3
+          with:
+            script: |
+              if ('refs/heads/main' === '${{github.ref}}') {
+                core.setFailed('Forbidden branch')
+              }
+        - name: Checkout detection-rules
+          uses: actions/checkout@v3
+          with:
+            path: detection-rules
+            fetch-depth: 0
 
-      - name: Check commit message
-        id: check_commit
-        env:
-          COMMIT_HASH: "${{github.event.inputs.commit_hash}}"
-        run: |
-          cd detection-rules
-          COMMIT_MESSAGE=$(git show -s --format=%B $COMMIT_HASH | grep "Lock versions for releases" || true)
-          if [ -z "$COMMIT_MESSAGE" ]; then
-            echo "::set-output name=check_message::false"
-          else
-            echo "::set-output name=check_message::true"
-          fi
-        shell: bash
+        - name: Extract version lock commit hash
+          run: |
+            cd detection-rules
+            COMMIT_HASH=$(git log --grep='Lock versions for releases' -1 --format='%H')
+            echo "COMMIT_HASH=$COMMIT_HASH" >> $GITHUB_ENV
+            echo "Extracted commit hash: $COMMIT_HASH"
 
-  fleet-pr:
-    name: Fleet PR
-    needs: check-commit
-    if: needs.check-commit.outputs.is_locked_commit == 'true'
-    runs-on: ubuntu-latest
-    steps:
-      - name: Validate the source branch
-        uses: actions/github-script@v3
-        with:
-          script: |
-            if ('refs/heads/main' === '${{github.ref}}') {
-              core.setFailed('Forbidden branch')
-            }
+        - name: Checkout commit hash
+          run: |
+            cd detection-rules
+            echo "Current branch is $GITHUB_REF"
+            echo "Checking out commit hash $COMMIT_HASH"
+            git checkout $COMMIT_HASH
 
-      - name: Checkout detection-rules
-        uses: actions/checkout@v3
-        with:
-          path: detection-rules
-          fetch-depth: 0
+        - name: Checkout elastic/integrations
+          uses: actions/checkout@v3
+          with:
+            token: ${{ secrets.READ_WRITE_RELEASE_FLEET }}
+            repository: ${{github.event.inputs.target_repo}}
+            path: integrations
 
-      - name: Checkout elastic/integrations
-        uses: actions/checkout@v3
-        with:
-          token: ${{ secrets.READ_WRITE_RELEASE_FLEET }}
-          repository: ${{github.event.inputs.target_repo}}
-          path: integrations
+        - name: Set up Python 3.8
+          uses: actions/setup-python@v2
+          with:
+            python-version: 3.8
 
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.8
+        - name: Install Python dependencies
+          run: |
+            cd detection-rules
+            python -m pip install --upgrade pip
+            pip cache purge
+            pip install .[dev]
 
-      - name: Install Python dependencies
-        run: |
-          cd detection-rules
-          python -m pip install --upgrade pip
-          pip cache purge
-          pip install .[dev]
+        - name: Bump prebuilt rules package version
+          env:
+            PACKAGE_MATURITY: "${{github.event.inputs.package_maturity}}"
+            NEW_PACKAGE: "${{github.event.inputs.new_package}}"
+          run: |
+            cd detection-rules
+            python -m detection_rules dev bump-pkg-versions \
+              --patch-release                               \
+              --new-package $NEW_PACKAGE                    \
+              --maturity $PACKAGE_MATURITY
 
-      - name: Checkout commit hash
-        env:
-          COMMIT_HASH: ${{github.event.inputs.commit_hash}}
-        run: |
-          cd detection-rules
-          git checkout $COMMIT_HASH
+        - name: Store release tag
+          if: github.event.inputs.package_maturity == 'ga'
+          run: |
+            cd detection-rules
+            output=$(cat detection_rules/etc/packages.yml | grep -oP '(?<=\sversion: )\S+')
+            echo "pkg_version=$output" >> $GITHUB_ENV
 
-      - name: Bump prebuilt rules package version
-        env:
-          PACKAGE_MATURITY: "${{github.event.inputs.package_maturity}}"
-          NEW_PACKAGE: "${{github.event.inputs.new_package}}"
-        run: |
-          cd detection-rules
-          python -m detection_rules dev bump-pkg-versions \
-            --patch-release                               \
-            --new-package $NEW_PACKAGE                    \
-            --maturity $PACKAGE_MATURITY
+        - name: Create release tag
+          if: github.event.inputs.package_maturity == 'ga'
+          run: |
+            cd detection-rules
+            RELEASE_TAG="integration-v${{ env.pkg_version }}"
+            echo "Creating release tag: $RELEASE_TAG"
+            git tag $RELEASE_TAG
+            git push origin $RELEASE_TAG
 
-      - name: Store release tag
-        if: ${{github.event.inputs.package_maturity}} == "ga"
-        id: packages-version
-        run: |
-          cd detection-rules
-          output=$(cat detection_rules/etc/packages.yml | grep -oP '(?<=\sversion: )\S+')
-          echo "::set-output name=pkg_version::$output"
+        - name: Build release package
+          run: |
+            cd detection-rules
+            python -m detection_rules dev build-release
 
-      - name: Create release tag
-        if: ${{github.event.inputs.package_maturity}} == "ga"
-        env:
-          RELEASE_TAG: "integration-v${{ steps.packages-version.outputs.pkg_version }}"
-        run: |
-          cd detection-rules
-          git tag $RELEASE_TAG
-          git push origin $RELEASE_TAG
+        - name: Set github config
+          run: |
+            git config --global user.email "72879786+protectionsmachine@users.noreply.github.com"
+            git config --global user.name "protectionsmachine"
 
-      - name: Build release package
-        env:
-          HISTORICAL: "${{github.event.inputs.add_historical}}"
-        run: |
-          cd detection-rules
-          python -m detection_rules dev build-release --add-historical $HISTORICAL
+        - name: Setup go
+          uses: actions/setup-go@v3
+          with:
+            go-version: '^1.20.1'
+            check-latest: true
 
-      - name: Set github config
-        run: |
-          git config --global user.email "72879786+protectionsmachine@users.noreply.github.com"
-          git config --global user.name "protectionsmachine"
+        - name: Build elastic-package
+          run: |
+            go install github.com/elastic/elastic-package@latest
 
-      - name: Setup go
-        uses: actions/setup-go@v3
-        with:
-          go-version: '^1.20.1'
-          check-latest: true
+        - name: Create the PR to Integrations
+          env:
+            DRAFT_ARGS: "${{startsWith(github.event.inputs.draft,'y') && '--draft' || ' '}}"
+            TARGET_REPO: "${{github.event.inputs.target_repo}}"
+            TARGET_BRANCH: "${{github.event.inputs.target_branch}}"
+            LOCAL_REPO: "../integrations"
+            GITHUB_TOKEN: "${{ secrets.READ_WRITE_RELEASE_FLEET }}"
+          run: |
+            cd detection-rules
+            python -m detection_rules dev integrations-pr \
+              $LOCAL_REPO                                 \
+              --github-repo $TARGET_REPO                  \
+              --base-branch $TARGET_BRANCH                \
+              --assign ${{github.actor}}                  \
+              $DRAFT_ARGS
 
-      - name: Build elastic-package
-        run: |
-          go install github.com/elastic/elastic-package@latest
-
-      - name: Create the PR to Integrations
-        env:
-          DRAFT_ARGS: "${{startsWith(github.event.inputs.draft,'y') && '--draft' || ' '}}"
-          TARGET_REPO: "${{github.event.inputs.target_repo}}"
-          TARGET_BRANCH: "${{github.event.inputs.target_branch}}"
-          LOCAL_REPO: "../integrations"
-          GITHUB_TOKEN: "${{ secrets.READ_WRITE_RELEASE_FLEET }}"
-        run: |
-          cd detection-rules
-          python -m detection_rules dev integrations-pr \
-            $LOCAL_REPO                                 \
-            --github-repo $TARGET_REPO                  \
-            --base-branch $TARGET_BRANCH                \
-            --assign ${{github.actor}}                  \
-            $DRAFT_ARGS
-
-      - name: Archive production artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: release-files
-          path: |
-            detection-rules/releases
+        - name: Archive production artifacts
+          uses: actions/upload-artifact@v3
+          with:
+            name: release-files
+            path: |
+              detection-rules/releases

detection_rules/devtools.py

@@ -86,13 +86,14 @@ def dev_group():
 @click.option('--update-version-lock', '-u', is_flag=True,
               help='Save version.lock.json file with updated rule versions in the package')
 @click.option('--generate-navigator', is_flag=True, help='Generate ATT&CK navigator files')
-@click.option('--add-historical', type=str, required=True, default="no",
+@click.option('--add-historical', type=str, required=True, default="yes",
               help='Generate historical package-registry files')
 @click.option('--update-message', type=str, help='Update message for new package')
 def build_release(config_file, update_version_lock: bool, generate_navigator: bool, add_historical: str,
                   update_message: str, release=None, verbose=True):
     """Assemble all the rules into Kibana-ready release files."""
     config = load_dump(config_file)['package']
+    registry_data = config['registry_data']
     add_historical = True if add_historical == "yes" else False
 
     if generate_navigator:
@@ -111,17 +112,18 @@ def build_release(config_file, update_version_lock: bool, generate_navigator: bo
     package.save(verbose=verbose)
 
     if add_historical:
-        previous_pkg_version = find_latest_integration_version("security_detection_engine", "ga", config['name'])
+        previous_pkg_version = find_latest_integration_version("security_detection_engine", "ga",
+                                                               registry_data['conditions']['kibana.version'].strip("^"))
         sde = SecurityDetectionEngine()
         historical_rules = sde.load_integration_assets(previous_pkg_version)
         historical_rules = sde.transform_legacy_assets(historical_rules)
 
-        docs = IntegrationSecurityDocsMDX(config['registry_data']['version'], Path(f'releases/{config["name"]}-docs'),
+        docs = IntegrationSecurityDocsMDX(registry_data['version'], Path(f'releases/{config["name"]}-docs'),
                                           True, historical_rules, package, note=update_message)
         docs.generate()
 
         click.echo(f'[+] Adding historical rules from {previous_pkg_version} package')
-        package.add_historical_rules(historical_rules, config['registry_data']['version'])
+        package.add_historical_rules(historical_rules, registry_data['version'])
 
     if verbose:
         package.get_package_hash(verbose=verbose)
@@ -207,7 +209,7 @@ def bump_versions(major_release: bool, minor_release: bool, patch_release: bool,
         pkg_data["registry_data"]["version"] = str(pkg_ver.bump_minor().bump_prerelease("beta"))
     if patch_release:
         latest_patch_release_ver = find_latest_integration_version("security_detection_engine",
-                                                                   maturity, pkg_data["name"])
+                                                                   maturity, pkg_kibana_ver)
 
         # if an existing minor or major does not have a package, bump from the last
         # example is 8.10.0-beta.1 is last, but on 9.0.0 major
@@ -219,13 +221,14 @@ def bump_versions(major_release: bool, minor_release: bool, patch_release: bool,
 
         if maturity == "ga":
             pkg_data["registry_data"]["version"] = str(latest_patch_release_ver.bump_patch())
-            pkg_data["registry_data"]["release"] = maturity
         else:
             # passing in true or false from GH actions; not using eval() for security purposes
             if new_package == "true":
                 latest_patch_release_ver = latest_patch_release_ver.bump_patch()
             pkg_data["registry_data"]["version"] = str(latest_patch_release_ver.bump_prerelease("beta"))
-            pkg_data["registry_data"]["release"] = maturity
+
+        if 'release' in pkg_data['registry_data']:
+            pkg_data['registry_data']['release'] = maturity
 
     click.echo(f"Kibana version: {pkg_data['name']}")
     click.echo(f"Package Kibana version: {pkg_data['registry_data']['conditions']['kibana.version']}")