From 284886292b38db132e53d9e8af022b579df894c8 Mon Sep 17 00:00:00 2001 From: Josh Rickard Date: Fri, 12 May 2023 16:33:47 -0500 Subject: [PATCH] Atomic Red Team - JSON Schema Validation CI (#2303) * feat: Adding atomic-red-team JSON Schema defintions * feat: Adding validate.py script to validate all atomics against the defined schema * feat: Adding validate-schema GitHub Workflow action to validate on every push to the repo * ci: Updated the validate-schema workflow to support and use Ruby instead of python * fix: Updated schema to remove schema draft version (not necessarily needed) and update to remove elevation_required as a required defined property * fix: Removed the yaml schema version * docs: Adding start of README * fix: Adding an updated/better version of the python validation but may ultimately be removed * feat: Adding Ruby version of validate.rb script * fix: Removing files not needed since we are changing to github action and using the new validation code * fix: Adding the yaml schema file back and removed the json version * docs: Updated README with documentation * fix: Updating schema to use new format validator * fix: Updated validate.rb to verify that the Technique IDs are in the correct format. * fix: Upating validate.rb to raise execptions so that failures flow up to the GitHub Action workflow * fix: Updated all tests that have input_arguments not conformaing to schema defintion for type value of path * fix: Updating the Validaton README for typos * fixL: Minor updates to the schema * minor schema changes * github actions fix * schema changes --------- Co-authored-by: MSAdministrator Co-authored-by: Carrie Roberts Co-authored-by: Hare Sudhan --- .github/workflows/validate-atomics.yml | 25 ++-- atomics/T1040/T1040.yaml | 2 +- atomics/T1048.003/T1048.003.yaml | 2 +- atomics/T1055.001/T1055.001.yaml | 2 +- atomics/T1059.003/T1059.003.yaml | 2 +- atomics/T1059.004/T1059.004.yaml | 2 +- atomics/T1078.004/T1078.004.yaml | 10 +- atomics/T1087.002/T1087.002.yaml | 4 +- atomics/T1110.001/T1110.001.yaml | 4 +- atomics/T1113/T1113.yaml | 2 +- atomics/T1136.003/T1136.003.yaml | 4 +- atomics/T1137.004/T1137.004.yaml | 2 +- atomics/T1187/T1187.yaml | 2 +- atomics/T1486/T1486.yaml | 6 +- atomics/T1531/T1531.yaml | 24 ++-- atomics/T1543.003/T1543.003.yaml | 8 +- atomics/T1548.002/T1548.002.yaml | 4 +- atomics/T1552.001/T1552.001.yaml | 2 +- atomics/T1572/T1572.yaml | 2 +- atomics/T1611/T1611.yaml | 2 +- bin/validate-atomics.rb | 31 ----- bin/validate/README.md | 167 +++++++++++++++++++++++ bin/validate/atomic-red-team.schema.yaml | 151 ++++++++++++++++++++ bin/validate/validate.py | 40 ++++++ poetry.lock | 80 ++++++++++- pyproject.toml | 1 + 26 files changed, 495 insertions(+), 86 deletions(-) delete mode 100755 bin/validate-atomics.rb create mode 100644 bin/validate/README.md create mode 100644 bin/validate/atomic-red-team.schema.yaml create mode 100644 bin/validate/validate.py diff --git a/.github/workflows/validate-atomics.yml b/.github/workflows/validate-atomics.yml index 96a44264..cef47e5c 100644 --- a/.github/workflows/validate-atomics.yml +++ b/.github/workflows/validate-atomics.yml @@ -1,25 +1,30 @@ name: validate-atomics on: push: - branches: [ master ] + branches: [master] pull_request: - branches: [ master ] + branches: [master] jobs: validate-atomics: runs-on: ubuntu-latest steps: - name: checkout repo - uses: actions/checkout@v2 - - - name: setup ruby - uses: ruby/setup-ruby@v1 + uses: actions/checkout@v3 + - name: Install poetry + run: pipx install poetry + - name: setup python3.11 + uses: actions/setup-python@v4 + id: setup-python with: - ruby-version: 2.7 - bundler-cache: true + python-version: "3.11.2" + cache: "poetry" + + - name: Install dependencies + run: poetry install --no-interaction --no-root - name: validate the format of atomics tests against the spec run: | - bin/validate-atomics.rb + poetry run python bin/validate/validate.py validate-terraform: runs-on: ubuntu-latest @@ -30,4 +35,4 @@ jobs: - name: Terraform fmt id: fmt run: terraform fmt -recursive -check - continue-on-error: false + continue-on-error: false \ No newline at end of file diff --git a/atomics/T1040/T1040.yaml b/atomics/T1040/T1040.yaml index 0b7769eb..7e7323f0 100644 --- a/atomics/T1040/T1040.yaml +++ b/atomics/T1040/T1040.yaml @@ -74,7 +74,7 @@ atomic_tests: type: url default: https://1.eu.dl.wireshark.org/win64/Wireshark-win64-latest.exe tshark_path: - description: path to tshark.exe + description: path to tshark.exe type: path default: c:\program files\wireshark\tshark.exe npcap_url: diff --git a/atomics/T1048.003/T1048.003.yaml b/atomics/T1048.003/T1048.003.yaml index aea3ba2b..8041baba 100644 --- a/atomics/T1048.003/T1048.003.yaml +++ b/atomics/T1048.003/T1048.003.yaml @@ -181,7 +181,7 @@ atomic_tests: default: dlpuser ftp_port: description: Your FTP's port - type: string + type: integer default: 21 dependency_executor_name: powershell dependencies: diff --git a/atomics/T1055.001/T1055.001.yaml b/atomics/T1055.001/T1055.001.yaml index 067f3b70..30292e26 100644 --- a/atomics/T1055.001/T1055.001.yaml +++ b/atomics/T1055.001/T1055.001.yaml @@ -13,7 +13,7 @@ atomic_tests: input_arguments: process_id: description: PID of input_arguments - type: integer + type: string default: (Start-Process notepad -PassThru).id dll_payload: description: DLL to Inject diff --git a/atomics/T1059.003/T1059.003.yaml b/atomics/T1059.003/T1059.003.yaml index d49f7394..8dd2bb78 100644 --- a/atomics/T1059.003/T1059.003.yaml +++ b/atomics/T1059.003/T1059.003.yaml @@ -86,7 +86,7 @@ atomic_tests: default: $env:temp\T1059_003note.txt max_to_print: description: The maximum number of Wordpad windows the test will open/print. - type: string + type: integer default: 75 dependency_executor_name: powershell dependencies: diff --git a/atomics/T1059.004/T1059.004.yaml b/atomics/T1059.004/T1059.004.yaml index 14e41909..62dbb67e 100644 --- a/atomics/T1059.004/T1059.004.yaml +++ b/atomics/T1059.004/T1059.004.yaml @@ -208,7 +208,7 @@ atomic_tests: input_arguments: remote_url: description: url of remote payload - type: Url + type: url default: https://raw.githubusercontent.com/redcanaryco/atomic-red-team/master/atomics/T1059.004/src/pipe-to-shell.sh dependency_executor_name: bash dependencies: diff --git a/atomics/T1078.004/T1078.004.yaml b/atomics/T1078.004/T1078.004.yaml index 0ced90f0..2e1683f5 100644 --- a/atomics/T1078.004/T1078.004.yaml +++ b/atomics/T1078.004/T1078.004.yaml @@ -59,23 +59,23 @@ atomic_tests: input_arguments: username: description: Azure username - type: String + type: string default: null password: description: Azure password - type: String + type: string default: null resource_group: description: Name of the resource group - type: String + type: string default: null runbook_name: description: Name of the runbook name - type: String + type: string default: null automation_account_name: description: Name of the automation account name - type: String + type: string default: null dependency_executor_name: powershell dependencies: diff --git a/atomics/T1087.002/T1087.002.yaml b/atomics/T1087.002/T1087.002.yaml index 4a9149b7..aa00e78c 100644 --- a/atomics/T1087.002/T1087.002.yaml +++ b/atomics/T1087.002/T1087.002.yaml @@ -194,7 +194,7 @@ atomic_tests: default: $env:UserDnsDomain uac_prop: description: UAC Property to search - type: string + type: integer default: 524288 dependencies: - description: | @@ -313,7 +313,7 @@ atomic_tests: cd $env:temp .\kerbrute.exe userenum -d #{Domain} --dc #{DomainController} $env:TEMP\username.txt name: powershell -- name: Wevtutil - Discover NTLM Users Remote +- name: Wevtutil - Discover NTLM Users Remote auto_generated_guid: b8a563d4-a836-4993-a74e-0a19b8481bfe description: | This test discovers users who have authenticated against a Domain Controller via NTLM. diff --git a/atomics/T1110.001/T1110.001.yaml b/atomics/T1110.001/T1110.001.yaml index b12f4404..794600d5 100644 --- a/atomics/T1110.001/T1110.001.yaml +++ b/atomics/T1110.001/T1110.001.yaml @@ -164,7 +164,7 @@ atomic_tests: input_arguments: remote_url: description: url of remote payload - type: Url + type: url default: https://raw.githubusercontent.com/redcanaryco/atomic-red-team/master/atomics/T1110.001/src/sudo_bruteforce.sh dependency_executor_name: bash dependencies: @@ -200,7 +200,7 @@ atomic_tests: input_arguments: remote_url: description: url of remote payload - type: Url + type: url default: https://raw.githubusercontent.com/redcanaryco/atomic-red-team/master/atomics/T1110.001/src/sudo_bruteforce.sh dependency_executor_name: bash dependencies: diff --git a/atomics/T1113/T1113.yaml b/atomics/T1113/T1113.yaml index 1832a6bc..0b3aea7c 100644 --- a/atomics/T1113/T1113.yaml +++ b/atomics/T1113/T1113.yaml @@ -106,7 +106,7 @@ atomic_tests: default: c:\temp\T1113_desktop.zip recording_time: description: Time to take screenshots - type: string + type: integer default: 5 executor: name: powershell diff --git a/atomics/T1136.003/T1136.003.yaml b/atomics/T1136.003/T1136.003.yaml index 8533b4d7..1f0225ab 100644 --- a/atomics/T1136.003/T1136.003.yaml +++ b/atomics/T1136.003/T1136.003.yaml @@ -38,7 +38,7 @@ atomic_tests: default: "atomicredteam" userprincipalname: description: User principal name (UPN) for the new Azure user being created format email address - type: String + type: string default: "atomicredteam@yourdomain.com" password: description: Password for the new Azure AD user being created @@ -75,7 +75,7 @@ atomic_tests: default: "atomicredteam" userprincipalname: description: User principal name (UPN) for the new Azure user being created format email address - type: String + type: string default: "atomicredteam@yourdomain.com" password: description: Password for the new Azure AD user being created diff --git a/atomics/T1137.004/T1137.004.yaml b/atomics/T1137.004/T1137.004.yaml index 8a97184d..3df08f44 100644 --- a/atomics/T1137.004/T1137.004.yaml +++ b/atomics/T1137.004/T1137.004.yaml @@ -18,7 +18,7 @@ atomic_tests: default: file://PathToAtomicsFolder\T1137.004\src\T1137.004.html outlook_version: description: Version of Outlook that is installed - type: string + type: float default: 16.0 # Microsoft 365: 16.0 # Outlook 2019: 16.0 diff --git a/atomics/T1187/T1187.yaml b/atomics/T1187/T1187.yaml index cbd84bbf..5f9d0da2 100644 --- a/atomics/T1187/T1187.yaml +++ b/atomics/T1187/T1187.yaml @@ -18,7 +18,7 @@ atomic_tests: default: 10.0.0.2 efsApi: description: EFS API to use to coerce authentication - type: string + type: integer default: 1 petitpotam_path: description: PetitPotam Windows executable diff --git a/atomics/T1486/T1486.yaml b/atomics/T1486/T1486.yaml index 81a988bc..2d9c67b0 100644 --- a/atomics/T1486/T1486.yaml +++ b/atomics/T1486/T1486.yaml @@ -186,17 +186,17 @@ atomic_tests: auto_generated_guid: 4541e2c2-33c8-44b1-be79-9161440f1718 description: Gpg4win is a Windows tool (also called Kleopatra which is the preferred certificate manager) that uses email and file encryption packages for symmetric encryption. - It is used by attackers to encrypt disks. User will need to add pass phrase to encrypt file as automation is not allowed under newer versions. + It is used by attackers to encrypt disks. User will need to add pass phrase to encrypt file as automation is not allowed under newer versions. supported_platforms: - windows input_arguments: GPG_Exe_Location: description: Path of the GPG program - type: Path + type: path default: 'C:\Program Files (x86)\GnuPG\bin\gpg.exe' File_to_Encrypt_Location: description: Path of File - type: Path + type: path default: '$env:temp\test.txt' dependencies: - description: | diff --git a/atomics/T1531/T1531.yaml b/atomics/T1531/T1531.yaml index b847f027..65d39a78 100644 --- a/atomics/T1531/T1531.yaml +++ b/atomics/T1531/T1531.yaml @@ -98,7 +98,7 @@ atomic_tests: input_arguments: user_account: description: User account whose password will be changed. - type: String + type: string default: ARTUser executor: command: | @@ -114,12 +114,12 @@ atomic_tests: input_arguments: user_account: description: User account which will be deleted. - type: String + type: string default: ARTUser user_password: description: User password. - type: String - default: ARTPassword + type: string + default: ARTPassword executor: command: | dscl . -delete /Users/#{user_account} #enter admin password @@ -140,15 +140,15 @@ atomic_tests: input_arguments: user_account: description: User account which will be deleted. - type: String + type: string default: ARTUserAccount user_name: description: New user name. - type: String + type: string default: ARTUser user_password: description: New user password. - type: String + type: string default: ARTPassword executor: command: | @@ -156,7 +156,7 @@ atomic_tests: cleanup_command: | sysadminctl -addUser #{user_account} -fullName "#{user_name}" -password #{user_password} name: sh - elevation_required: true + elevation_required: true - name: Azure AD - Delete user via Azure AD PowerShell auto_generated_guid: 4f577511-dc1c-4045-bcb8-75d2457f01f4 description: Deletes a user in Azure AD. Adversaries may interrupt availability of system and network resources by inhibiting access to accounts utilized by legitimate users. Accounts may be deleted, locked, or manipulated (excluding changed credentials) to remove access to accounts. @@ -165,7 +165,7 @@ atomic_tests: input_arguments: userprincipalname: description: User principal name (UPN) for the Azure user being deleted - type: String + type: string default: "atomicredteam@yourdomain.com" dependency_executor_name: powershell dependencies: @@ -179,7 +179,7 @@ atomic_tests: command: |- Connect-AzureAD $userprincipalname = "#{userprincipalname}" - Remove-AzureADUser -ObjectId $userprincipalname + Remove-AzureADUser -ObjectId $userprincipalname cleanup_command: N/A name: powershell - name: Azure AD - Delete user via Azure CLI @@ -190,7 +190,7 @@ atomic_tests: input_arguments: userprincipalname: description: User principal name (UPN) for the Azure user being deleted - type: String + type: string default: "atomicredteam@yourdomain.com" dependency_executor_name: powershell dependencies: @@ -200,7 +200,7 @@ atomic_tests: - description: Check if Azure CLI is installed and install via PowerShell prereq_command: az account list get_prereq_command: echo "use the following to install the Azure CLI $ProgressPreference = 'SilentlyContinue'; Invoke-WebRequest -Uri https://aka.ms/installazurecliwindows -OutFile .\AzureCLI.msi; Start-Process msiexec.exe -Wait -ArgumentList '/I AzureCLI.msi /quiet'; Remove-Item .\AzureCLI.msi" - - description: Update the userprincipalname to meet your requirements + - description: Update the userprincipalname to meet your requirements prereq_command: Update the input arguments so the userprincipalname value is accurate for your environment get_prereq_command: echo "Update the input arguments in the .yaml file so that the userprincipalname value is accurate for your environment" executor: diff --git a/atomics/T1543.003/T1543.003.yaml b/atomics/T1543.003/T1543.003.yaml index 6d434156..5712fa28 100644 --- a/atomics/T1543.003/T1543.003.yaml +++ b/atomics/T1543.003/T1543.003.yaml @@ -33,11 +33,11 @@ atomic_tests: default: PathToAtomicsFolder\T1543.003\bin\AtomicService.exe service_type: description: Type of service. May be own|share|interact|kernel|filesys|rec|userown|usershare - type: String + type: string default: Own startup_type: description: Service start method. May be boot|system|auto|demand|disabled|delayed-auto - type: String + type: string default: auto service_name: description: Name of the Service @@ -142,11 +142,11 @@ atomic_tests: default: PathToAtomicsFolder\T1543.003\bin\AtomicService.exe service_type: description: Type of service. May be own,share,interact,kernel,filesys,rec,userown,usershare - type: String + type: string default: Own startup_type: description: Service start method. May be boot,system,auto,demand,disabled,delayed-auto - type: String + type: string default: auto service_name: description: Name of the Service diff --git a/atomics/T1548.002/T1548.002.yaml b/atomics/T1548.002/T1548.002.yaml index 2464a919..80ecc3c5 100644 --- a/atomics/T1548.002/T1548.002.yaml +++ b/atomics/T1548.002/T1548.002.yaml @@ -134,13 +134,13 @@ atomic_tests: supported_platforms: - windows input_arguments: - command.to.execute: + command_to_execute: description: Command to execute type: string default: cmd.exe /c notepad.exe executor: command: | - New-Item -Force -Path "HKCU:\Software\Classes\Folder\shell\open\command" -Value '#{command.to.execute}' + New-Item -Force -Path "HKCU:\Software\Classes\Folder\shell\open\command" -Value '#{command_to_execute}' New-ItemProperty -Force -Path "HKCU:\Software\Classes\Folder\shell\open\command" -Name "DelegateExecute" Start-Process -FilePath $env:windir\system32\sdclt.exe Start-Sleep -s 3 diff --git a/atomics/T1552.001/T1552.001.yaml b/atomics/T1552.001/T1552.001.yaml index 0f617ed9..1aa05d91 100644 --- a/atomics/T1552.001/T1552.001.yaml +++ b/atomics/T1552.001/T1552.001.yaml @@ -81,7 +81,7 @@ atomic_tests: input_arguments: file_path: description: Path to search - type: String + type: string default: /home executor: name: bash diff --git a/atomics/T1572/T1572.yaml b/atomics/T1572/T1572.yaml index d1d4f0fe..d36418c9 100644 --- a/atomics/T1572/T1572.yaml +++ b/atomics/T1572/T1572.yaml @@ -25,7 +25,7 @@ atomic_tests: query_volume: description: Number of DNS queries to send type: integer - default: "1000" + default: 1000 domain: description: Default domain to simulate against type: string diff --git a/atomics/T1611/T1611.yaml b/atomics/T1611/T1611.yaml index 45fd52fd..f55fa9f5 100644 --- a/atomics/T1611/T1611.yaml +++ b/atomics/T1611/T1611.yaml @@ -93,7 +93,7 @@ atomic_tests: listen_port: description: TCP Port to listen on for callback from the host system. - type: string + type: integer default: 4444 dependency_executor_name: sh diff --git a/bin/validate-atomics.rb b/bin/validate-atomics.rb deleted file mode 100755 index 4c1b78f7..00000000 --- a/bin/validate-atomics.rb +++ /dev/null @@ -1,31 +0,0 @@ -#! /usr/bin/env ruby -$LOAD_PATH << "#{File.dirname(File.dirname(__FILE__))}/atomic_red_team" unless $LOAD_PATH.include? "#{File.dirname(File.dirname(__FILE__))}/atomic_red_team" -require 'yaml' -require 'atomic_red_team' - -ATOMIC_RED_TEAM = AtomicRedTeam.new -ATOMIC_TEST_TEMPLATE = "#{File.dirname(File.dirname(__FILE__))}/atomic_red_team/atomic_test_template.yaml" -USED_GUIDS_FILE = "#{File.dirname(File.dirname(__FILE__))}/atomics/used_guids.txt" - -oks = [] -fails = [] -unique_guid_array = [] - -ATOMIC_RED_TEAM.atomic_test_paths.each do |path| - begin - print "Validating #{path}..." - AtomicRedTeam.new.validate_atomic_yaml!(YAML.load_file(path), USED_GUIDS_FILE, unique_guid_array) - - oks << path - puts "OK" - rescue => ex - fails << path - puts "FAIL\n#{ex}\n" - # puts "FAIL\n#{ex}\n#{ex.backtrace.join("\n")})" - end -end - -puts -puts "#{oks.count + fails.count} techniques, #{fails.count} failures" - -exit fails.count \ No newline at end of file diff --git a/bin/validate/README.md b/bin/validate/README.md new file mode 100644 index 00000000..2d6cf2d0 --- /dev/null +++ b/bin/validate/README.md @@ -0,0 +1,167 @@ +# Validaton + +We provide validation of each defined Atomic Red Team test in the form of a [JSON Schema](https://json-schema.org/). This schema defines the structure and format of an Atomic test. + +- [Validaton](#validaton) + - [Validation Requirements](#validation-requirements) + - [atomic\_tests](#atomic_tests) + - [input\_arguments](#input_arguments) + - [dependencies](#dependencies) + - [dependency\_executor\_name](#dependency_executor_name) + - [executor](#executor) + - [Tooling \& Usage](#tooling--usage) + - [Error Messages (Ruby Version)](#error-messages-ruby-version) + - [Error Messages (Python version)](#error-messages-python-version) + +We use this schema to validate the format of Atomics using a [GitHub Action](.../.github/workflows/validate-schema.yml) which runs on every push to the repository. If an Atomic fails validation, it is not allowed to be merged into the main branch. + +``` +📦atomics + ┣ 📂T1234 + ┃ ┣ 📂T1234.md + ┃ ┗ 📂T1234.yaml <-- This is where all the atomic tests live + ┃ ┣ 📂src + ┃ ┃ ┣ 📜payload1.sct <-- A paload file needed by one of the T1234 atomics (human readable) + ┃ ┃ ┣ 📜payload2.dll <-- Another payload file needed by one of the T1234 atomics (binary) + ``` +In general, a set of atomic tests for a technique should never depend on payloads or supporting files from other atomic directories. We want to keep things nice and close. Use git symlinks if you really need to share files between techniques. + +Atomic tests should be fully automated whenever possible, requiring no continued interaction. Include any needed options to execute the commands seamlessly, for example SysInternal's -accepteula option or any -q or -quiet modes. + +## Validation Requirements + +To explain the requirements around validation, we have broken down each main component. + +Each yaml specification requires the following main level entities: + +* attack_techniques - A Mitre ATT&CK Technique or Sub-Technique ID with a capital T. +* display_name - Name of the technique or sub-technique as defined by ATT&CK. +* atomic_tests - One or more Atomic tests for a technique / sub-technique. + +### atomic_tests + +Each `atomic_test` object must have the following fields defined: + +|Property Name|Description|Data Type|Accepted Values| +|-------------|-----------|---------|---------------| +|name |The name of the test.|String|Any| +|description |A description about the test|String|Any| +|supported_platforms|One or more supported operating system platforms for this test. This is a list of supported_platforms and each must be unique.|List[String]|windows, macos, linux, office-365, azure-ad, google-workspace, saas, iaas, containers, iaas:gcp, iaas:azure, iaas:aws| + +### input_arguments + +Each defined test can supply one or more `input_arguments`. Please note that input arguments are not required and only optional. If you do provide a `input_argument` then each must be unique and contain a unique named property as well as sub-properties. + +If your argument requires a String or null value you can use the following properties. + +|Property Name|Description|Data Type|Accepted Values| +|-------------|-----------|---------|---------------| +|{unique_name}|A unique name for the input argument that will be referenced in commands|String|`^[a-zA-Z0-9_-]+$`| +|description |A description about the the input argument property|String|Any| +|type |The data type of the value for this property|String|Path, Url, String (please note the capitalization)| +|default |The default value for the argument|String or Null|Any| + +If your argument requires a integer or float you can use the following properties. + +|Property Name|Description|Data Type|Accepted Values| +|-------------|-----------|---------|---------------| +|{unique_name}|A unique name for the input argument that will be referenced in commands|String|`^[a-zA-Z0-9_-]+$`| +|description |A description about the the input argument property|String|Any| +|type |The data type of the value for this property|String|Integer, Float (please note the capitalization)| +|default |The default value for the argument|Number or Null|Any| + +### dependencies + +A list of dependies that must be met to successfully run this atomic. This is optional but if provided you must provide the following values for that dependency. + +> You can supply more than 1 dependency. The `dependencies` property takes a list of dependencies. + +|Property Name|Description|Data Type|Accepted Values| +|-------------|-----------|---------|---------------| +|description |A description about the the input argument property|String|Any| +|prereq_command|Commands to check if prerequisites for running this test are met. For the "command_prompt" executor, if any command returns a non-zero exit code, the pre-requisites are not met. For the "powershell" executor, all commands are run as a script block and the script block must return 0 for success.|String|Any| +|get_prereq_command|Commands to meet this prerequisite or a message describing how to meet this prereq|String|Any| + + +### dependency_executor_name + +The executor for the prereq commands, defaults to the same executor used by the attack commands. This field is optional but must be one of the following values: + +* command_prompt +* powershell +* sh +* bash +* manual + +### executor + +The `executor` propery contains a list of unique executors for each environment that the test belongs to or has defined definitions for. + +Each defined `executor` can define the following properties. + +|Property Name|Description|Data Type|Accepted Values| +|-------------|-----------|---------|---------------| +|name |The name of the executor to use to execute this command sequence.|String|command_prompt, sh, bash, powershell, aws, az, gcloud, kubectl| +|command |The command string to execute.|String|Any| + +Each executor can also have the following fields, but these are only specified when needed. + +|Property Name|Description|Data Type|Accepted Values| +|-------------|-----------|---------|---------------| +|elevation_required|indicates whether command must be run with admin privileges.|Bool|true or false| +|cleanup_command |The command string to execute to cleanup the system after executing the command above.|String|Any| + +You can also specify the executor type of `manual`. The manual executor requires another field called `steps` which is a list of manual steps that the user must take to perform an action. + +## Tooling & Usage + +There are two main entrypoints to validate atomics. You can do so manually by cloning the repository and running the [validate.rb](validate.rb). + +```ruby +ruby ./bin/validate/validate.rb +``` + +Additionally, the validation script will run on each push to the repository using the provided GitHub Action. + +## Error Messages (Ruby Version) + +TODO + +## Error Messages (Python version) + +A typical error message when validation fails looks like the following: + +```bash +{'description': 'Daily scheduled task execution time', 'type': 'string', 'default': '07:45'} is not valid under any of the given schemas + +Failed validating 'anyOf' in schema['properties']['atomic_tests']['items']['properties']['input_arguments']['patternProperties']['^[a-zA-Z0-9]*$']: + {'anyOf': [{'properties': {'default': {'type': ['string', 'null']}, + 'description': {'type': 'string'}, + 'type': {'enum': ['Path', 'Url', 'String'], + 'type': 'string'}}, + 'type': 'object'}, + {'properties': {'default': {'type': ['number', 'null']}, + 'description': {'type': 'string'}, + 'type': {'enum': ['Integer', 'Float'], + 'type': 'string'}}, + 'required': ['description', 'type', 'default'], + 'type': 'object'}], + 'type': 'object'} + +On instance['atomic_tests'][6]['input_arguments']['time']: + {'default': '07:45', + 'description': 'Daily scheduled task execution time', + 'type': 'string'} +``` + +With this error, it may be unclear exactly why the validation failed. That is why we have formatted the output to parse this error to make it more readable. For example, the parsed version is + +```bash +Error occurred with ./atomics/T1053.005/T1053.005.yaml. +Each of the following are why it failed: + + 'string' is not one of ['Path', 'Url', 'String'] + +The JSON Path is $.atomic_tests[6].input_arguments.time +``` + diff --git a/bin/validate/atomic-red-team.schema.yaml b/bin/validate/atomic-red-team.schema.yaml new file mode 100644 index 00000000..13f4fff1 --- /dev/null +++ b/bin/validate/atomic-red-team.schema.yaml @@ -0,0 +1,151 @@ +$id: https://json-schema.org/draft/2020-12/schema +title: Atomic Schema +description: A schema for atomics within the atomic-red-team project +type: object +properties: + attack_technique: + description: A MITRE ATT&CK Technique ID with a capital T + type: string + format: technique_id + pattern: T[\.\d]{4,8} + display_name: + description: Name of the technique as defined by ATT&CK. + type: string + atomic_tests: + description: One or more Atomic tests for a technique + type: array + items: + $ref: "#/$defs/test" + minItems: 1 + uniqueItems: true +$defs: + test: + type: object + required: + - name + - description + - supported_platforms + - executor + properties: + name: + type: string + description: The name of the test. + auto_generated_guid: + type: string + description: A unique test GUID + description: + type: string + description: A description about the test + supported_platforms: + type: array + description: One or more supported operating system platforms for this test + uniqueItems: true + items: + type: string + enum: + - windows + - macos + - linux + - office-365 + - azure-ad + - google-workspace + - saas + - iaas + - containers + - iaas:gcp + - iaas:azure + - iaas:aws + input_arguments: + type: object + additionalProperties: false + properties: + "/": {} + patternProperties: + "^[\\w-]+$": + type: integer + type: object + required: + - description + properties: + description: + type: string + anyOf: + - required: + - type + properties: + type: + type: string + enum: + - integer + - float + default: + type: + - number + - "null" + - required: + - type + properties: + type: + type: string + enum: + - path + - url + - string + default: + type: + - string + - "null" + dependency_executor_name: + type: string + enum: + - command_prompt + - powershell + - sh + - bash + - manual + dependencies: + type: array + unique: true + items: + type: object + properties: + description: + type: string + prereq_command: + type: string + get_prereq_command: + type: string + required: + - description + - prereq_command + - get_prereq_command + executor: + type: object + required: + - name + properties: + name: + type: string + enum: + - command_prompt + - powershell + - sh + - bash + - manual + oneOf: + - required: + - command + properties: + elevation_required: + type: boolean + command: + type: string + cleanup_command: + type: + - string + - "null" + - required: + - steps + properties: + steps: + type: string diff --git a/bin/validate/validate.py b/bin/validate/validate.py new file mode 100644 index 00000000..e1fc4370 --- /dev/null +++ b/bin/validate/validate.py @@ -0,0 +1,40 @@ +"""Validates atomics based on JSON Schema.""" +import glob +import os.path +import sys +import yaml +from jsonschema import validate +from jsonschema.exceptions import ValidationError + +is_exception = False +with open(f"{os.path.dirname(os.path.abspath(__file__))}/atomic-red-team.schema.yaml", "r") as f: + schema = yaml.safe_load(f) + + for item in glob.glob("./atomics/T*/T*.yaml"): + with open(item, 'r') as file: + data = yaml.safe_load(file) + try: + validate( + instance=data, + schema=schema + ) + except ValidationError as ve: + print(f"Error occurred with {item}") + print("Each of the following are why it failed:") + if (context := ve.context) and len(context) > 0: + print(f"\n\t{context[0].message}\n") + else: + print(f"\n\t{ve}\n") + print(f"The JSON Path is {ve.json_path}") + is_exception = True + except Exception as e: + print(f"Error occurred with {item}") + print("Each of the following are why it failed:") + print(f"\n\t{e}\n") + is_exception = True + +if is_exception: + print("Validation Failed") + sys.exit(1) +else: + print("Validation Successful") \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index 8eeeddd5..bcf36497 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,80 @@ -# This file is automatically @generated by Poetry 1.4.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.4.2 and should not be changed by hand. + +[[package]] +name = "attrs" +version = "23.1.0" +description = "Classes Without Boilerplate" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.1.0-py3-none-any.whl", hash = "sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04"}, + {file = "attrs-23.1.0.tar.gz", hash = "sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[docs,tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] + +[[package]] +name = "jsonschema" +version = "4.17.3" +description = "An implementation of JSON Schema validation for Python" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "jsonschema-4.17.3-py3-none-any.whl", hash = "sha256:a870ad254da1a8ca84b6a2905cac29d265f805acc57af304784962a2aa6508f6"}, + {file = "jsonschema-4.17.3.tar.gz", hash = "sha256:0f864437ab8b6076ba6707453ef8f98a6a0d512a80e93f8abdb676f737ecb60d"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0,<0.17.0 || >0.17.0,<0.17.1 || >0.17.1,<0.17.2 || >0.17.2" + +[package.extras] +format = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3987", "uri-template", "webcolors (>=1.11)"] +format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "uri-template", "webcolors (>=1.11)"] + +[[package]] +name = "pyrsistent" +version = "0.19.3" +description = "Persistent/Functional/Immutable data structures" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pyrsistent-0.19.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:20460ac0ea439a3e79caa1dbd560344b64ed75e85d8703943e0b66c2a6150e4a"}, + {file = "pyrsistent-0.19.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4c18264cb84b5e68e7085a43723f9e4c1fd1d935ab240ce02c0324a8e01ccb64"}, + {file = "pyrsistent-0.19.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4b774f9288dda8d425adb6544e5903f1fb6c273ab3128a355c6b972b7df39dcf"}, + {file = "pyrsistent-0.19.3-cp310-cp310-win32.whl", hash = "sha256:5a474fb80f5e0d6c9394d8db0fc19e90fa540b82ee52dba7d246a7791712f74a"}, + {file = "pyrsistent-0.19.3-cp310-cp310-win_amd64.whl", hash = "sha256:49c32f216c17148695ca0e02a5c521e28a4ee6c5089f97e34fe24163113722da"}, + {file = "pyrsistent-0.19.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:f0774bf48631f3a20471dd7c5989657b639fd2d285b861237ea9e82c36a415a9"}, + {file = "pyrsistent-0.19.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3ab2204234c0ecd8b9368dbd6a53e83c3d4f3cab10ecaf6d0e772f456c442393"}, + {file = "pyrsistent-0.19.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e42296a09e83028b3476f7073fcb69ffebac0e66dbbfd1bd847d61f74db30f19"}, + {file = "pyrsistent-0.19.3-cp311-cp311-win32.whl", hash = "sha256:64220c429e42a7150f4bfd280f6f4bb2850f95956bde93c6fda1b70507af6ef3"}, + {file = "pyrsistent-0.19.3-cp311-cp311-win_amd64.whl", hash = "sha256:016ad1afadf318eb7911baa24b049909f7f3bb2c5b1ed7b6a8f21db21ea3faa8"}, + {file = "pyrsistent-0.19.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c4db1bd596fefd66b296a3d5d943c94f4fac5bcd13e99bffe2ba6a759d959a28"}, + {file = "pyrsistent-0.19.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aeda827381f5e5d65cced3024126529ddc4289d944f75e090572c77ceb19adbf"}, + {file = "pyrsistent-0.19.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:42ac0b2f44607eb92ae88609eda931a4f0dfa03038c44c772e07f43e738bcac9"}, + {file = "pyrsistent-0.19.3-cp37-cp37m-win32.whl", hash = "sha256:e8f2b814a3dc6225964fa03d8582c6e0b6650d68a232df41e3cc1b66a5d2f8d1"}, + {file = "pyrsistent-0.19.3-cp37-cp37m-win_amd64.whl", hash = "sha256:c9bb60a40a0ab9aba40a59f68214eed5a29c6274c83b2cc206a359c4a89fa41b"}, + {file = "pyrsistent-0.19.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:a2471f3f8693101975b1ff85ffd19bb7ca7dd7c38f8a81701f67d6b4f97b87d8"}, + {file = "pyrsistent-0.19.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cc5d149f31706762c1f8bda2e8c4f8fead6e80312e3692619a75301d3dbb819a"}, + {file = "pyrsistent-0.19.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3311cb4237a341aa52ab8448c27e3a9931e2ee09561ad150ba94e4cfd3fc888c"}, + {file = "pyrsistent-0.19.3-cp38-cp38-win32.whl", hash = "sha256:f0e7c4b2f77593871e918be000b96c8107da48444d57005b6a6bc61fb4331b2c"}, + {file = "pyrsistent-0.19.3-cp38-cp38-win_amd64.whl", hash = "sha256:c147257a92374fde8498491f53ffa8f4822cd70c0d85037e09028e478cababb7"}, + {file = "pyrsistent-0.19.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:b735e538f74ec31378f5a1e3886a26d2ca6351106b4dfde376a26fc32a044edc"}, + {file = "pyrsistent-0.19.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:99abb85579e2165bd8522f0c0138864da97847875ecbd45f3e7e2af569bfc6f2"}, + {file = "pyrsistent-0.19.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3a8cb235fa6d3fd7aae6a4f1429bbb1fec1577d978098da1252f0489937786f3"}, + {file = "pyrsistent-0.19.3-cp39-cp39-win32.whl", hash = "sha256:c74bed51f9b41c48366a286395c67f4e894374306b197e62810e0fdaf2364da2"}, + {file = "pyrsistent-0.19.3-cp39-cp39-win_amd64.whl", hash = "sha256:878433581fc23e906d947a6814336eee031a00e6defba224234169ae3d3d6a98"}, + {file = "pyrsistent-0.19.3-py3-none-any.whl", hash = "sha256:ccf0d6bd208f8111179f0c26fdf84ed7c3891982f2edaeae7422575f47e66b64"}, + {file = "pyrsistent-0.19.3.tar.gz", hash = "sha256:1a2994773706bbb4995c31a97bc94f1418314923bd1048c6d964837040376440"}, +] [[package]] name = "pyyaml" @@ -53,4 +129,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "849e6d6d7360f5ed35d66cb6fb3bd11ec904da8b76a61511a183d6a2e01a153b" +content-hash = "794303f69da5b50dda744446c5f9c78700a726f087531aea762153d5e620ae69" diff --git a/pyproject.toml b/pyproject.toml index 05b1314e..32340f98 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,6 +9,7 @@ packages = [{include = "atomic_red_team"}] [tool.poetry.dependencies] python = "^3.11" pyyaml = "^6.0" +jsonschema = "^4.17.3" [build-system]