From d24c1e280012e442e2e95111da463f1527efca8d Mon Sep 17 00:00:00 2001 From: Thomas Patzke Date: Sun, 29 Mar 2020 13:25:04 +0200 Subject: [PATCH 1/6] CI testing with GitHub Actions --- .github/workflows/sigma-test.yml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .github/workflows/sigma-test.yml diff --git a/.github/workflows/sigma-test.yml b/.github/workflows/sigma-test.yml new file mode 100644 index 000000000..eee872989 --- /dev/null +++ b/.github/workflows/sigma-test.yml @@ -0,0 +1,29 @@ +# This workflow will install Python dependencies, run tests and lint with a single version of Python +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Sigma Tools and Rule Tests + +on: + push: + branches: + - "*" + pull_request: + branches: [ master ] + +jobs: + test-sigma: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.8 + uses: actions/setup-python@v1 + with: + python-version: 3.8 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r tools/requirements.txt -r tools/requirements-devel.txt -r tools/requirements-misp.txt -r tests/requirements-test.txt + - name: Test with Makefile + run: | + make test + make test-backend-es-qs From fbe40bd1e842fcdbba57ab1ec4947ee3a53d80ea Mon Sep 17 00:00:00 2001 From: Thomas Patzke Date: Sun, 29 Mar 2020 13:41:03 +0200 Subject: [PATCH 2/6] Fixed Elasticsearch test * Splitted into separate action * Install dependencies --- .github/workflows/sigma-test.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sigma-test.yml b/.github/workflows/sigma-test.yml index eee872989..291d54683 100644 --- a/.github/workflows/sigma-test.yml +++ b/.github/workflows/sigma-test.yml @@ -23,7 +23,14 @@ jobs: run: | python -m pip install --upgrade pip pip install -r tools/requirements.txt -r tools/requirements-devel.txt -r tools/requirements-misp.txt -r tests/requirements-test.txt - - name: Test with Makefile + - name: Test Sigma Tools and Rules run: | make test + - name: Test Generated Elasticsearch Query Strings + run: | + wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | apt-key add - + apt install -y apt-transport-https + echo "deb https://artifacts.elastic.co/packages/7.x/apt stable main" >> /etc/apt/sources.list.d/elastic.list + apt update + apt install -y elasticsearch make test-backend-es-qs From 821a631325edf0a397e4a1ba1129dc5ecee995bc Mon Sep 17 00:00:00 2001 From: Thomas Patzke Date: Sun, 29 Mar 2020 14:00:15 +0200 Subject: [PATCH 3/6] Run Elasticsearch installation as root --- .github/workflows/sigma-test.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/sigma-test.yml b/.github/workflows/sigma-test.yml index 291d54683..d83359492 100644 --- a/.github/workflows/sigma-test.yml +++ b/.github/workflows/sigma-test.yml @@ -28,9 +28,9 @@ jobs: make test - name: Test Generated Elasticsearch Query Strings run: | - wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | apt-key add - - apt install -y apt-transport-https - echo "deb https://artifacts.elastic.co/packages/7.x/apt stable main" >> /etc/apt/sources.list.d/elastic.list - apt update - apt install -y elasticsearch + wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add - + sudo apt install -y apt-transport-https + echo "deb https://artifacts.elastic.co/packages/7.x/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic.list + sudo apt update + sudo apt install -y elasticsearch make test-backend-es-qs From d68b900077e4913ef1cbbe0c2dfdf7ec87bf1785 Mon Sep 17 00:00:00 2001 From: Thomas Patzke Date: Sun, 29 Mar 2020 14:37:27 +0200 Subject: [PATCH 4/6] Wait for Elasticsearch before running tests --- .github/workflows/sigma-test.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sigma-test.yml b/.github/workflows/sigma-test.yml index d83359492..1fcb71769 100644 --- a/.github/workflows/sigma-test.yml +++ b/.github/workflows/sigma-test.yml @@ -32,5 +32,8 @@ jobs: sudo apt install -y apt-transport-https echo "deb https://artifacts.elastic.co/packages/7.x/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic.list sudo apt update - sudo apt install -y elasticsearch + sudo apt install -y elasticsearch curl jq + sudo systemctl start elasticsearch + until curl -s elasticsearch:9200; do sleep 1; done + until curl -s elasticsearch:9200/_cluster/health | jq -e '.status == "green"'; do sleep 1; done make test-backend-es-qs From 5e258efbe74970e489b3883d83a8c17f5fed02ac Mon Sep 17 00:00:00 2001 From: Thomas Patzke Date: Sun, 29 Mar 2020 14:57:34 +0200 Subject: [PATCH 5/6] Improved Elasticsearch waiting process --- .github/workflows/sigma-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sigma-test.yml b/.github/workflows/sigma-test.yml index 1fcb71769..f356f8ef8 100644 --- a/.github/workflows/sigma-test.yml +++ b/.github/workflows/sigma-test.yml @@ -34,6 +34,6 @@ jobs: sudo apt update sudo apt install -y elasticsearch curl jq sudo systemctl start elasticsearch - until curl -s elasticsearch:9200; do sleep 1; done - until curl -s elasticsearch:9200/_cluster/health | jq -e '.status == "green"'; do sleep 1; done + until curl -s elasticsearch:9200; do echo "Waiting for Elasticsearch server..."; sleep 10; done + until curl -s elasticsearch:9200/_cluster/health | jq -e '.status != "red"'; do echo "Waiting for Elasticsearch cluster to become ready..."; sleep 10; done make test-backend-es-qs From 4dbe5e2f172b1c29c5d28a7d83ce1eba749f023a Mon Sep 17 00:00:00 2001 From: Thomas Patzke Date: Sun, 29 Mar 2020 15:19:13 +0200 Subject: [PATCH 6/6] Moved Elasticsearch dependencies to generic dependencies Omitting waiting for Elasticsearch as it should be started at this time. --- .github/workflows/sigma-test.yml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/sigma-test.yml b/.github/workflows/sigma-test.yml index f356f8ef8..3b4aff849 100644 --- a/.github/workflows/sigma-test.yml +++ b/.github/workflows/sigma-test.yml @@ -23,17 +23,15 @@ jobs: run: | python -m pip install --upgrade pip pip install -r tools/requirements.txt -r tools/requirements-devel.txt -r tools/requirements-misp.txt -r tests/requirements-test.txt + wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add - + sudo apt install -y apt-transport-https + echo "deb https://artifacts.elastic.co/packages/7.x/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic.list + sudo apt update + sudo apt install -y elasticsearch + sudo systemctl start elasticsearch - name: Test Sigma Tools and Rules run: | make test - name: Test Generated Elasticsearch Query Strings run: | - wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add - - sudo apt install -y apt-transport-https - echo "deb https://artifacts.elastic.co/packages/7.x/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic.list - sudo apt update - sudo apt install -y elasticsearch curl jq - sudo systemctl start elasticsearch - until curl -s elasticsearch:9200; do echo "Waiting for Elasticsearch server..."; sleep 10; done - until curl -s elasticsearch:9200/_cluster/health | jq -e '.status != "red"'; do echo "Waiting for Elasticsearch cluster to become ready..."; sleep 10; done make test-backend-es-qs