Sergey Polzunov
1fb60d6475
* first pass * Adding a dedicated code checking workflow * Type fixes * linting config and python version bump * Type hints * Drop incorrect config option * More fixes * Style fixes * CI adjustments * Pyproject fixes * CI & pyproject fixes * Proper version bump * Tests formatting * Resolve cirtular dependency * Test fixes * Make sure the tests are formatted correctly * Check tweaks * Bumping python version in CI images * Pin marshmallow do 3.x because 4.x is not supported * License fix * Convert path to str * Making myself a codeowner * Missing kwargs param * Adding a missing kwargs to `set_score` * Update .github/CODEOWNERS Co-authored-by: Mika Ayenson, PhD <Mikaayenson@users.noreply.github.com> * Dropping unnecessary raise * Dropping skipped test * Drop unnecessary var * Drop unused commented-out func * Disable typehinting for the whole func * Update linting command * Invalid type hist on the input param * Incorrect field type * Incorrect value used fix * Stricter values check * Simpler function call * Type condition fix * TOML formatter fix * Simpligy output conditions * Formatting * Use proper types instead of aliases * MITRE attack fixes * Using pathlib.Path for an argument * Use proper method to update a set from a dict * First round of `ruff` fixes * More fixes * More fixes * Hack against cyclic dependency * Ignore `PLC0415` * Remove unused markers * Cleanup * Fixing the incorrect condition * Update .github/CODEOWNERS Co-authored-by: Mika Ayenson, PhD <Mikaayenson@users.noreply.github.com> * Set explicit default values for optional fields * Update the guidelines * Adding None Defaults --------- Co-authored-by: Mika Ayenson, PhD <Mikaayenson@users.noreply.github.com> Co-authored-by: eric-forte-elastic <eric.forte@elastic.co>
71 lines
2.0 KiB
Python
71 lines
2.0 KiB
Python
# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
|
# or more contributor license agreements. Licensed under the Elastic License
|
|
# 2.0; you may not use this file except in compliance with the Elastic License
|
|
# 2.0.
|
|
|
|
"""Detection Rules tests."""
|
|
|
|
import json
|
|
import os
|
|
import pathlib
|
|
|
|
from detection_rules.eswrap import combine_sources
|
|
|
|
CURRENT_DIR = pathlib.Path(__file__).resolve().parent
|
|
DATA_DIR = CURRENT_DIR / "data"
|
|
TP_DIR = DATA_DIR / "true_positives"
|
|
FP_DIR = DATA_DIR / "false_positives"
|
|
|
|
|
|
def get_fp_dirs():
|
|
"""Get a list of fp dir names."""
|
|
return FP_DIR.glob("*")
|
|
|
|
|
|
def get_fp_data_files():
|
|
"""get FP data files by fp dir name."""
|
|
data = {}
|
|
for fp_dir in get_fp_dirs():
|
|
path = pathlib.Path(fp_dir)
|
|
fp_dir_name = path.name
|
|
relative_dir_name = pathlib.Path("false_positives") / fp_dir_name
|
|
data[fp_dir_name] = combine_sources(*get_data_files(relative_dir_name).values())
|
|
|
|
return data
|
|
|
|
|
|
def get_data_files_list(*folder, ext="ndjson", recursive=False):
|
|
"""Get TP or FP file list."""
|
|
folder = os.path.sep.join(folder)
|
|
data_dir = pathlib.Path(DATA_DIR) / folder
|
|
|
|
glob = "**" if recursive else ""
|
|
glob += f"*.{ext}"
|
|
|
|
return data_dir.glob(glob)
|
|
|
|
|
|
def get_data_files(*folder, ext="ndjson", recursive=False):
|
|
"""Get data from data files."""
|
|
data_files = {}
|
|
for data_file in get_data_files_list(*folder, ext=ext, recursive=recursive):
|
|
path = pathlib.Path(data_file)
|
|
with path.open() as f:
|
|
file_name = path.stem
|
|
|
|
if ext in (".ndjson", ".jsonl"):
|
|
data = f.readlines()
|
|
data_files[file_name] = [json.loads(d) for d in data]
|
|
else:
|
|
data_files[file_name] = json.load(f)
|
|
|
|
return data_files
|
|
|
|
|
|
def get_data_file(*folder):
|
|
path = pathlib.Path(DATA_DIR) / os.path.sep.join(folder)
|
|
if path.exists():
|
|
with path.open() as f:
|
|
return json.load(f)
|
|
return None
|