From c165783fffd8814f18616c76ec48878726d19188 Mon Sep 17 00:00:00 2001 From: wagga40 <6437862+wagga40@users.noreply.github.com> Date: Thu, 12 Aug 2021 15:26:46 +0200 Subject: [PATCH 1/3] Add an option to enhance default output by choosing fields Add an option to output in JSON or YAML --- Pipfile | 1 + Pipfile.lock | 265 +++++++++++++++++++++++++++--------------- tools/sigma/sigmac.py | 61 ++++++++-- 3 files changed, 225 insertions(+), 102 deletions(-) diff --git a/Pipfile b/Pipfile index 060d74e88..718c305f6 100644 --- a/Pipfile +++ b/Pipfile @@ -20,6 +20,7 @@ urllib3 = "~=1.26" progressbar2 = "~=3.47" pymisp = "~=2.4.123" PyYAML = "~=5.1" +"ruamel.yaml" = "*" [requires] python_version = "3.8" diff --git a/Pipfile.lock b/Pipfile.lock index f83fca957..2416e3258 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "9d6e50bfd41bb3de5ebbae350555fe4b67c24e2c186aac053905a7740a69e8b2" + "sha256": "08bbbed72c177a3a7a43aff79af8fdde3a0ac42e15d7e112d64cac2c5d5b6e68" }, "pipfile-spec": 6, "requires": { @@ -21,6 +21,7 @@ "sha256:149e90d6d8ac20db7a955ad60cf0e6881a3f20d37096140088356da6c716b0b1", "sha256:ef6aaac3ca6cd92904cdd0d83f629a15f18053ec84e6432106f7a4d04ae4f5fb" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "version": "==21.2.0" }, "certifi": { @@ -30,26 +31,29 @@ ], "version": "==2021.5.30" }, - "chardet": { + "charset-normalizer": { "hashes": [ - "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa", - "sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5" + "sha256:0c8911edd15d19223366a194a513099a302055a962bca2cec0f54b8b63175d8b", + "sha256:f23667ebe1084be45f6ae0538e4a5a865206544097e4e8bbcacf42cd02a348f3" ], - "version": "==4.0.0" + "markers": "python_version >= '3'", + "version": "==2.0.4" }, "deprecated": { "hashes": [ "sha256:08452d69b6b5bc66e8330adde0a4f8642e969b9e1702904d137eeb29c8ffc771", "sha256:6d2de2de7931a968874481ef30208fd4e08da39177d61d3d4ebdf4366e7dbca1" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.2.12" }, "idna": { "hashes": [ - "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6", - "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0" + "sha256:14475042e284991034cb48e06f6851428fb14c4dc953acd9be9a5e95c7b6dd7a", + "sha256:467fbad99067910785144ce333826c71fb0e63a425657295239737f7ecd125f3" ], - "version": "==2.10" + "markers": "python_version >= '3'", + "version": "==3.2" }, "jsonschema": { "hashes": [ @@ -68,24 +72,46 @@ }, "pymisp": { "hashes": [ - "sha256:7ab159ba589f54d105c59cb990722369c57d8f587b5df215a79ed4059cb57b8a", - "sha256:c6496a6884fe3a671e9dd3c314564b4e94b8827845f5ea0004ab3649373e9db2" + "sha256:5971eba9a4d3b7f5ee47035417c7692fc0ec45d581afcaa63e3f7e2d6a400923", + "sha256:641e3db1af1010cff3a652df6eb51ac4f4e540b1801b811d5e009c59114bf26a" ], "index": "pypi", - "version": "==2.4.141.1" + "version": "==2.4.148" }, "pyrsistent": { "hashes": [ - "sha256:2e636185d9eb976a18a8a8e96efce62f2905fea90041958d8cc2a189756ebf3e" + "sha256:097b96f129dd36a8c9e33594e7ebb151b1515eb52cceb08474c10a5479e799f2", + "sha256:2aaf19dc8ce517a8653746d98e962ef480ff34b6bc563fc067be6401ffb457c7", + "sha256:404e1f1d254d314d55adb8d87f4f465c8693d6f902f67eb6ef5b4526dc58e6ea", + "sha256:48578680353f41dca1ca3dc48629fb77dfc745128b56fc01096b2530c13fd426", + "sha256:4916c10896721e472ee12c95cdc2891ce5890898d2f9907b1b4ae0f53588b710", + "sha256:527be2bfa8dc80f6f8ddd65242ba476a6c4fb4e3aedbf281dfbac1b1ed4165b1", + "sha256:58a70d93fb79dc585b21f9d72487b929a6fe58da0754fa4cb9f279bb92369396", + "sha256:5e4395bbf841693eaebaa5bb5c8f5cdbb1d139e07c975c682ec4e4f8126e03d2", + "sha256:6b5eed00e597b5b5773b4ca30bd48a5774ef1e96f2a45d105db5b4ebb4bca680", + "sha256:73ff61b1411e3fb0ba144b8f08d6749749775fe89688093e1efef9839d2dcc35", + "sha256:772e94c2c6864f2cd2ffbe58bb3bdefbe2a32afa0acb1a77e472aac831f83427", + "sha256:773c781216f8c2900b42a7b638d5b517bb134ae1acbebe4d1e8f1f41ea60eb4b", + "sha256:a0c772d791c38bbc77be659af29bb14c38ced151433592e326361610250c605b", + "sha256:b29b869cf58412ca5738d23691e96d8aff535e17390128a1a52717c9a109da4f", + "sha256:c1a9ff320fa699337e05edcaae79ef8c2880b52720bc031b219e5b5008ebbdef", + "sha256:cd3caef37a415fd0dae6148a1b6957a8c5f275a62cca02e18474608cb263640c", + "sha256:d5ec194c9c573aafaceebf05fc400656722793dac57f254cd4741f3c27ae57b4", + "sha256:da6e5e818d18459fa46fac0a4a4e543507fe1110e808101277c5a2b5bab0cd2d", + "sha256:e79d94ca58fcafef6395f6352383fa1a76922268fa02caa2272fff501c2fdc78", + "sha256:f3ef98d7b76da5eb19c37fda834d50262ff9167c65658d1d8f974d2e4d90676b", + "sha256:f4c8cabb46ff8e5d61f56a037974228e978f26bfefce4f61a4b1ac0ba7a2ab72" ], - "version": "==0.17.3" + "markers": "python_version >= '3.6'", + "version": "==0.18.0" }, "python-dateutil": { "hashes": [ - "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c", - "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a" + "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", + "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" ], - "version": "==2.8.1" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.8.2" }, "python-utils": { "hashes": [ @@ -131,26 +157,62 @@ }, "requests": { "hashes": [ - "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804", - "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e" + "sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24", + "sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7" ], "index": "pypi", - "version": "==2.25.1" + "version": "==2.26.0" + }, + "ruamel.yaml": { + "hashes": [ + "sha256:106bc8d6dc6a0ff7c9196a47570432036f41d556b779c6b4e618085f57e39e67", + "sha256:ffb9b703853e9e8b7861606dfdab1026cf02505bade0653d1880f4b2db47f815" + ], + "index": "pypi", + "version": "==0.17.10" + }, + "ruamel.yaml.clib": { + "hashes": [ + "sha256:0847201b767447fc33b9c235780d3aa90357d20dd6108b92be544427bea197dd", + "sha256:1866cf2c284a03b9524a5cc00daca56d80057c5ce3cdc86a52020f4c720856f0", + "sha256:31ea73e564a7b5fbbe8188ab8b334393e06d997914a4e184975348f204790277", + "sha256:3fb9575a5acd13031c57a62cc7823e5d2ff8bc3835ba4d94b921b4e6ee664104", + "sha256:4ff604ce439abb20794f05613c374759ce10e3595d1867764dd1ae675b85acbd", + "sha256:72a2b8b2ff0a627496aad76f37a652bcef400fd861721744201ef1b45199ab78", + "sha256:78988ed190206672da0f5d50c61afef8f67daa718d614377dcd5e3ed85ab4a99", + "sha256:7b2927e92feb51d830f531de4ccb11b320255ee95e791022555971c466af4527", + "sha256:7f7ecb53ae6848f959db6ae93bdff1740e651809780822270eab111500842a84", + "sha256:825d5fccef6da42f3c8eccd4281af399f21c02b32d98e113dbc631ea6a6ecbc7", + "sha256:846fc8336443106fe23f9b6d6b8c14a53d38cef9a375149d61f99d78782ea468", + "sha256:89221ec6d6026f8ae859c09b9718799fea22c0e8da8b766b0b2c9a9ba2db326b", + "sha256:9efef4aab5353387b07f6b22ace0867032b900d8e91674b5d8ea9150db5cae94", + "sha256:a32f8d81ea0c6173ab1b3da956869114cae53ba1e9f72374032e33ba3118c233", + "sha256:a49e0161897901d1ac9c4a79984b8410f450565bbad64dbfcbf76152743a0cdb", + "sha256:ada3f400d9923a190ea8b59c8f60680c4ef8a4b0dfae134d2f2ff68429adfab5", + "sha256:bf75d28fa071645c529b5474a550a44686821decebdd00e21127ef1fd566eabe", + "sha256:cfdb9389d888c5b74af297e51ce357b800dd844898af9d4a547ffc143fa56751", + "sha256:d67f273097c368265a7b81e152e07fb90ed395df6e552b9fa858c6d2c9f42502", + "sha256:dc6a613d6c74eef5a14a214d433d06291526145431c3b964f5e16529b1842bed", + "sha256:de9c6b8a1ba52919ae919f3ae96abb72b994dd0350226e28f3686cb4f142165c" + ], + "markers": "python_version < '3.10' and platform_python_implementation == 'CPython'", + "version": "==0.2.6" }, "six": { "hashes": [ "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.16.0" }, "urllib3": { "hashes": [ - "sha256:753a0374df26658f99d826cfe40394a686d05985786d946fbe4165b5148f5a7c", - "sha256:a7acd0977125325f516bda9735fa7142b909a8d01e8b2e4c8108d0984e6e0098" + "sha256:39fb8672126159acb139a7718dd10806104dec1e2f0f6c88aab05d17df10c8d4", + "sha256:f57b4c16c62fa2760b7e3d97c35b255512fb6b59a259730f36ba32ce9f8e342f" ], "index": "pypi", - "version": "==1.26.5" + "version": "==1.26.6" }, "wrapt": { "hashes": [ @@ -200,6 +262,7 @@ "sha256:f881853d2643a29e643609da57b96d5f9c9b93f62429dcc1cbb413c7d07f0e1a", "sha256:fe60131d21b31fd1a14bd43e6bb88256f69dfc3188b3a89d736d6c71ed43ec95" ], + "markers": "python_version >= '3.6'", "version": "==3.7.4.post0" }, "antlr4-python3-runtime": { @@ -214,21 +277,23 @@ "sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f", "sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3" ], + "markers": "python_full_version >= '3.5.3'", "version": "==3.0.1" }, "attackcti": { "hashes": [ - "sha256:60059c597f39074db979482931c8771c31581c76e0ae6451c04214a1330a5d2f", - "sha256:a0c44c7065d2568b728e62a8325b0c5fde9d6901e4e0199bde7a9bab974bdcb9" + "sha256:2516b00631d4f0f8e05e950281ed94566774587b968901c02296e174835f0786", + "sha256:98d9c80a2c566847aa6d95fe824f48e8c45a418bbbb212e96dcf468693754cea" ], "index": "pypi", - "version": "==0.3.4.3" + "version": "==0.3.4.4" }, "attrs": { "hashes": [ "sha256:149e90d6d8ac20db7a955ad60cf0e6881a3f20d37096140088356da6c716b0b1", "sha256:ef6aaac3ca6cd92904cdd0d83f629a15f18053ec84e6432106f7a4d04ae4f5fb" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "version": "==21.2.0" }, "certifi": { @@ -243,8 +308,17 @@ "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa", "sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "version": "==4.0.0" }, + "charset-normalizer": { + "hashes": [ + "sha256:0c8911edd15d19223366a194a513099a302055a962bca2cec0f54b8b63175d8b", + "sha256:f23667ebe1084be45f6ae0538e4a5a865206544097e4e8bbcacf42cd02a348f3" + ], + "markers": "python_version >= '3'", + "version": "==2.0.4" + }, "colorama": { "hashes": [ "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b", @@ -313,11 +387,11 @@ }, "elasticsearch": { "hashes": [ - "sha256:9a77172be02bc4855210d83f0f1346a1e7d421e3cb2ca47ba81ac0c5a717b3a0", - "sha256:c67b0f6541eda6de9f92eaea319c070aa2710c5d4d4ee5e3dfa3c21bd95aa378" + "sha256:084979d21cc2955903ecc215bb40b8180207b2bcb5e52ec0ec7dd6f60affd01e", + "sha256:f3ab1454e646170bbc6796b8707e4bff125234391d2acc022221e1c0313becb4" ], "index": "pypi", - "version": "==7.12.0" + "version": "==7.14.0" }, "elasticsearch-async": { "hashes": [ @@ -329,16 +403,18 @@ }, "idna": { "hashes": [ - "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6", - "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0" + "sha256:14475042e284991034cb48e06f6851428fb14c4dc953acd9be9a5e95c7b6dd7a", + "sha256:467fbad99067910785144ce333826c71fb0e63a425657295239737f7ecd125f3" ], - "version": "==2.10" + "markers": "python_version >= '3'", + "version": "==3.2" }, "more-itertools": { "hashes": [ "sha256:2cf89ec599962f2ddc4d568a05defc40e0a587fbc10d5989713638864c36be4d", "sha256:83f0308e05477c68f56ea3a888172c78ed5d5b3c282addb67508e7ba6c8f813a" ], + "markers": "python_version >= '3.5'", "version": "==8.8.0" }, "multidict": { @@ -381,27 +457,30 @@ "sha256:f21756997ad8ef815d8ef3d34edd98804ab5ea337feedcd62fb52d22bf531281", "sha256:fc13a9524bc18b6fb6e0dbec3533ba0496bbed167c56d0aabefd965584557d80" ], + "markers": "python_version >= '3.6'", "version": "==5.1.0" }, "packaging": { "hashes": [ - "sha256:5b327ac1320dc863dca72f4514ecc086f31186744b84a230374cc1fd776feae5", - "sha256:67714da7f7bc052e064859c05c595155bd1ee9f69f76557e21f051443c20947a" + "sha256:7dc96269f53a4ccec5c0670940a4281106dd0bb343f47b7471f779df49c2fbe7", + "sha256:c86254f9220d55e31cc94d69bade760f0847da8000def4dfe1c6b872fd14ff14" ], - "version": "==20.9" + "markers": "python_version >= '3.6'", + "version": "==21.0" }, "pathspec": { "hashes": [ - "sha256:86379d6b86d75816baba717e64b1a3a3469deb93bb76d613c9ce79edc5cb68fd", - "sha256:aa0cb481c4041bf52ffa7b0d8fa6cd3e88a2ca4879c533c9153882ee2556790d" + "sha256:7d15c4ddb0b5c802d161efc417ec1a2558ea2653c2e8ad9c19098201dc1c993a", + "sha256:e564499435a2673d586f6b2130bb5b95f04a3ba06f81b8f895b651a3c76aabb1" ], - "version": "==0.8.1" + "version": "==0.9.0" }, "pluggy": { "hashes": [ "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0", "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.13.1" }, "py": { @@ -409,6 +488,7 @@ "sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3", "sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.10.0" }, "pyparsing": { @@ -416,6 +496,7 @@ "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1", "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b" ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.4.7" }, "pytest": { @@ -470,76 +551,70 @@ }, "requests": { "hashes": [ - "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804", - "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e" + "sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24", + "sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7" ], "index": "pypi", - "version": "==2.25.1" + "version": "==2.26.0" }, "simplejson": { "hashes": [ - "sha256:034550078a11664d77bc1a8364c90bb7eef0e44c2dbb1fd0a4d92e3997088667", - "sha256:05b43d568300c1cd43f95ff4bfcff984bc658aa001be91efb3bb21df9d6288d3", - "sha256:0dd9d9c738cb008bfc0862c9b8fa6743495c03a0ed543884bf92fb7d30f8d043", - "sha256:10fc250c3edea4abc15d930d77274ddb8df4803453dde7ad50c2f5565a18a4bb", - "sha256:2862beabfb9097a745a961426fe7daf66e1714151da8bb9a0c430dde3d59c7c0", - "sha256:292c2e3f53be314cc59853bd20a35bf1f965f3bc121e007ab6fd526ed412a85d", - "sha256:2d3eab2c3fe52007d703a26f71cf649a8c771fcdd949a3ae73041ba6797cfcf8", - "sha256:2e7b57c2c146f8e4dadf84977a83f7ee50da17c8861fd7faf694d55e3274784f", - "sha256:311f5dc2af07361725033b13cc3d0351de3da8bede3397d45650784c3f21fbcf", - "sha256:344e2d920a7f27b4023c087ab539877a1e39ce8e3e90b867e0bfa97829824748", - "sha256:3fabde09af43e0cbdee407555383063f8b45bfb52c361bc5da83fcffdb4fd278", - "sha256:42b8b8dd0799f78e067e2aaae97e60d58a8f63582939af60abce4c48631a0aa4", - "sha256:4b3442249d5e3893b90cb9f72c7d6ce4d2ea144d2c0d9f75b9ae1e5460f3121a", - "sha256:55d65f9cc1b733d85ef95ab11f559cce55c7649a2160da2ac7a078534da676c8", - "sha256:5c659a0efc80aaaba57fcd878855c8534ecb655a28ac8508885c50648e6e659d", - "sha256:72d8a3ffca19a901002d6b068cf746be85747571c6a7ba12cbcf427bfb4ed971", - "sha256:75ecc79f26d99222a084fbdd1ce5aad3ac3a8bd535cd9059528452da38b68841", - "sha256:76ac9605bf2f6d9b56abf6f9da9047a8782574ad3531c82eae774947ae99cc3f", - "sha256:7d276f69bfc8c7ba6c717ba8deaf28f9d3c8450ff0aa8713f5a3280e232be16b", - "sha256:7f10f8ba9c1b1430addc7dd385fc322e221559d3ae49b812aebf57470ce8de45", - "sha256:8042040af86a494a23c189b5aa0ea9433769cc029707833f261a79c98e3375f9", - "sha256:813846738277729d7db71b82176204abc7fdae2f566e2d9fcf874f9b6472e3e6", - "sha256:845a14f6deb124a3bcb98a62def067a67462a000e0508f256f9c18eff5847efc", - "sha256:869a183c8e44bc03be1b2bbcc9ec4338e37fa8557fc506bf6115887c1d3bb956", - "sha256:8acf76443cfb5c949b6e781c154278c059b09ac717d2757a830c869ba000cf8d", - "sha256:8f713ea65958ef40049b6c45c40c206ab363db9591ff5a49d89b448933fa5746", - "sha256:934115642c8ba9659b402c8bdbdedb48651fb94b576e3b3efd1ccb079609b04a", - "sha256:9551f23e09300a9a528f7af20e35c9f79686d46d646152a0c8fc41d2d074d9b0", - "sha256:9a2b7543559f8a1c9ed72724b549d8cc3515da7daf3e79813a15bdc4a769de25", - "sha256:a55c76254d7cf8d4494bc508e7abb993a82a192d0db4552421e5139235604625", - "sha256:ad8f41c2357b73bc9e8606d2fa226233bf4d55d85a8982ecdfd55823a6959995", - "sha256:af4868da7dd53296cd7630687161d53a7ebe2e63814234631445697bd7c29f46", - "sha256:afebfc3dd3520d37056f641969ce320b071bc7a0800639c71877b90d053e087f", - "sha256:b59aa298137ca74a744c1e6e22cfc0bf9dca3a2f41f51bc92eb05695155d905a", - "sha256:bc00d1210567a4cdd215ac6e17dc00cb9893ee521cee701adfd0fa43f7c73139", - "sha256:c1cb29b1fced01f97e6d5631c3edc2dadb424d1f4421dad079cb13fc97acb42f", - "sha256:c94dc64b1a389a416fc4218cd4799aa3756f25940cae33530a4f7f2f54f166da", - "sha256:ceaa28a5bce8a46a130cd223e895080e258a88d51bf6e8de2fc54a6ef7e38c34", - "sha256:cff6453e25204d3369c47b97dd34783ca820611bd334779d22192da23784194b", - "sha256:d0b64409df09edb4c365d95004775c988259efe9be39697d7315c42b7a5e7e94", - "sha256:d4813b30cb62d3b63ccc60dd12f2121780c7a3068db692daeb90f989877aaf04", - "sha256:da3c55cdc66cfc3fffb607db49a42448785ea2732f055ac1549b69dcb392663b", - "sha256:e058c7656c44fb494a11443191e381355388443d543f6fc1a245d5d238544396", - "sha256:fed0f22bf1313ff79c7fc318f7199d6c2f96d4de3234b2f12a1eab350e597c06", - "sha256:ffd4e4877a78c84d693e491b223385e0271278f5f4e1476a4962dca6824ecfeb" + "sha256:02bc0b7b643fa255048862f580bb4b7121b88b456bc64dabf9bf11df116b05d7", + "sha256:02c04b89b0a456a97d5313357dd9f2259c163a82c5307e39e7d35bb38d7fd085", + "sha256:05cd392c1c9b284bda91cf9d7b6f3f46631da459e8546fe823622e42cf4794bb", + "sha256:1331a54fda3c957b9136402943cf8ebcd29c0c92101ba70fa8c2fc9cdf1b8476", + "sha256:18302970ce341c3626433d4ffbdac19c7cca3d6e2d54b12778bcb8095f695473", + "sha256:1ebbaa48447b60a68043f58e612021e8893ebcf1662a1b18a2595ca262776d7e", + "sha256:2104475a0263ff2a3dffca214c9676eb261e90d06d604ac7063347bd289ac84c", + "sha256:23169d78f74fd25f891e89c779a63fcb857e66ab210096f4069a5b1c9e2dc732", + "sha256:32edf4e491fe174c54bf6682d794daf398736158d1082dbcae526e4a5af6890b", + "sha256:3904b528e3dc0facab73a4406ebf17f007f32f0a8d7f4c6aa9ed5cbad3ea0f34", + "sha256:391a8206e698557a4155354cf6996c002aa447a21c5c50fb94a0d26fd6cca586", + "sha256:3c80b343503da8b13fa7d48d1a2395be67e97b67a849eb79d88ad3b12783e7da", + "sha256:3dddd31857d8230aee88c24f485ebca36d1d875404b2ef11ac15fa3c8a01dc34", + "sha256:56f57c231cdd01b6a1c0532ea9088dff2afe7f4f4bda61c060bcb1a853e6b564", + "sha256:5b080be7de4c647fa84252cf565298a13842658123bd1a322a8c32b6359c8f1e", + "sha256:6285b91cfa37e024f372b9b77d14f279380eebc4f709db70c593c069602e1926", + "sha256:6510e886d9e9006213de2090c55f504b12f915178a2056b94840ed1d89abe68e", + "sha256:6ff6710b824947ef5a360a5a5ae9809c32cedc6110df3b64f01080c1bc1a1f08", + "sha256:79545a6d93bb38f86a00fbc6129cb091a86bb858e7d53b1aaa10d927d3b6732e", + "sha256:88a69c7e8059a4fd7aa2a31d2b3d89077eaae72eb741f18a32cb57d04018ff4c", + "sha256:8f174567c53413383b8b7ec2fbe88d41e924577bc854051f265d4c210cd72999", + "sha256:a52b80b9d1085db6e216980d1d28a8f090b8f2203a8c71b4ea13441bd7a2e86e", + "sha256:b25748e71c5df3c67b5bda2cdece373762d319cb5f773f14ae2f90dfb4320314", + "sha256:b45b5f6c9962953250534217b18002261c5b9383349b95fb0140899cdac2bf95", + "sha256:b4ed7b233e812ef1244a29fb0dfd3e149dbc34a2bd13b174a84c92d0cb580277", + "sha256:b60f48f780130f27f8d9751599925c3b78cf045f5d62dd918003effb65b45bda", + "sha256:c69a213ae72b75e8948f06a87d3675855bccb3037671222ffd235095e62f5a61", + "sha256:c91d0f2fc2ee1bd376f5a991c24923f12416d8c31a9b74a82c4b38b942fc2640", + "sha256:d61fb151be068127a0ce7758341cbe778495819622bc1e15eadf59fdb3a0481e", + "sha256:da72a452bcf4349fc467a12b54ab0e63e654a571cacc44084826d52bde12b6ee", + "sha256:dbcd6cd1a9abb5a13c5df93cdc5687f6877efcfefdc9350c22d4094dc4a7dd86", + "sha256:e056056718246c9cdd82d1e3d4ad854a7ceb057498bf994b529750a190a6bd98", + "sha256:e3aa10cce4053f3c1487aaf847a0faa4ae208e11f85a8e6f98de2291713a6616", + "sha256:e7433c604077a17dd71e8b29c96a15e486a70a97f4ed9c7f5e0df6e428af2f0b", + "sha256:f02db159e0afa9cb350f15f4f7b86755eae95267b9012ee90bde329aa643f76c", + "sha256:f32a703fe10cfc2d1020e296eeeeb650faa039678f6b79d9b820413a4c015ddc", + "sha256:fed5e862d9b501c5673c163c8593ebdb2c5422386089c529dfac28d70cd55858", + "sha256:ff7fe042169dd6fce8213c173a4c337f2e807ed5178093143c778eb0484c12ec" ], - "version": "==3.17.2" + "markers": "python_version >= '2.5' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==3.17.3" }, "six": { "hashes": [ "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.16.0" }, "stix2": { "hashes": [ - "sha256:15c9cf599f5c43124e76fe71b883e4918f6f4cf65b084c58ec64b6180f45c938", - "sha256:3ab60082e4bffb39f75ea9ddc338b64126ff1cd086e6173d39b860191ac26ff4" + "sha256:b9b2200e5c429a0a49d67c8902638d2f97df2ba4321e15dde067c5cb80c9e8e1" ], "index": "pypi", - "version": "==2.1.0" + "version": "==3.0.0" }, "stix2-patterns": { "hashes": [ @@ -565,11 +640,11 @@ }, "urllib3": { "hashes": [ - "sha256:753a0374df26658f99d826cfe40394a686d05985786d946fbe4165b5148f5a7c", - "sha256:a7acd0977125325f516bda9735fa7142b909a8d01e8b2e4c8108d0984e6e0098" + "sha256:39fb8672126159acb139a7718dd10806104dec1e2f0f6c88aab05d17df10c8d4", + "sha256:f57b4c16c62fa2760b7e3d97c35b255512fb6b59a259730f36ba32ce9f8e342f" ], "index": "pypi", - "version": "==1.26.5" + "version": "==1.26.6" }, "wcwidth": { "hashes": [ @@ -580,11 +655,10 @@ }, "yamllint": { "hashes": [ - "sha256:8a5f8e442f49309eaf3e9d7232ce76f2fc8026f5c0c0b164b83f33fed1399637", - "sha256:b0e4c89985c7f5f8451c2eb8c67d804d10ac13a4abe031cbf49bdf3465d01087" + "sha256:0b08a96750248fdf21f1e8193cb7787554ef75ed57b27f621cd6b3bf09af11a1" ], "index": "pypi", - "version": "==1.26.0" + "version": "==1.26.2" }, "yarl": { "hashes": [ @@ -626,6 +700,7 @@ "sha256:f0b059678fd549c66b89bed03efcabb009075bd131c248ecdf087bdb6faba24a", "sha256:fcbb48a93e8699eae920f8d92f7160c03567b421bc17362a9ffbbd706a816f71" ], + "markers": "python_version >= '3.6'", "version": "==1.6.3" } } diff --git a/tools/sigma/sigmac.py b/tools/sigma/sigmac.py index 0da400c2b..2e8863c16 100755 --- a/tools/sigma/sigmac.py +++ b/tools/sigma/sigmac.py @@ -18,6 +18,7 @@ import sys import argparse import yaml +import ruamel.yaml import json import pathlib import itertools @@ -53,10 +54,14 @@ ERR_RULE_FILTER_PARSING = 11 ERR_CONFIG_REQUIRED = 20 ERR_CONFIG_ORDER = 21 ERR_CONFIG_BACKEND = 22 +ERR_OUTPUT_FORMAT = 30 ERR_NOT_IMPLEMENTED = 42 ERR_PARTIAL_FIELD_MATCH = 80 ERR_FULL_FIELD_MATCH = 90 +# Allowed fields in output +allowed_fields = ["title", "id", "status", "description", "author", "references", "fields", "falsepositives", "level", "tags", "filename"] + def alliter(path): for sub in path.iterdir(): if sub.name.startswith("."): @@ -104,6 +109,11 @@ def set_argparser(): argparser.add_argument("--lists", "-l", action="store_true", help="List available output target formats and configurations") argparser.add_argument("--config", "-c", action="append", help="Configurations with field name and index mapping for target environment. Multiple configurations are merged into one. Last config is authoritative in case of conflicts.") argparser.add_argument("--output", "-o", default=None, help="Output file or filename prefix (if end with a '_','/' or '\\')") + argparser.add_argument("--output-fields", "-of", help="""Enhance your output with additional fields from the Sigma rule (not only the converted rule itself). + Select the fields you want by providing their list delimited with commas (no space). Only work with the '--output-format' option and with 'json' or 'yaml' value. + available additional fields : title, id, status, description, author, references, fields, falsepositives, level, tags. + This option do not have any effect for backends that already format output : elastalert, kibana, splukxml etc. """) + argparser.add_argument("--output-format", "-oF", default="raw", choices=["raw", "json", "yaml"], help="Use only if you want to have JSON or YAML output (default is raw text)") argparser.add_argument("--output-extention", "-e", default=None, help="Extention of Output file for filename prefix use") argparser.add_argument("--print0", action="store_true", help="Delimit results by NUL-character") argparser.add_argument("--backend-option", "-O", action="append", help="Options and switches that are passed to the backend") @@ -213,6 +223,18 @@ def main(): print("Sigma configuration parse error in %s: %s" % (conf_name, str(e)), file=sys.stderr) exit(ERR_CONFIG_PARSING) + if cmdargs.output_fields: + if cmdargs.output_format != "raw": + output_fields_rejected = [field for field in cmdargs.output_fields.split(",") if field not in allowed_fields] # Not allowed fields + if output_fields_rejected: + print("These fields are not allowed (check help for allow field list) : %s" % (", ".join(output_fields_rejected)), file=sys.stderr) + exit(ERR_OUTPUT_FORMAT) + else: + output_fields_filtered = [field for field in cmdargs.output_fields.split(",") if field in allowed_fields] # Keep only allowed fields + else: + print("The '--output-fields' or '-of' arguments must be use with '--output-format' or '-oF' equal to 'json' or 'yaml'", file=sys.stderr) + exit(ERR_OUTPUT_FORMAT) + backend_options = BackendOptions(cmdargs.backend_option, cmdargs.backend_config) backend = backend_class(sigmaconfigs, backend_options) @@ -240,6 +262,7 @@ def main(): out = sys.stdout error = 0 + output_array = [] for sigmafile in get_inputs(cmdargs.inputs, cmdargs.recurse): logger.debug("* Processing Sigma input %s" % (sigmafile)) try: @@ -249,12 +272,13 @@ def main(): f = sigmafile.open(encoding='utf-8') parser = SigmaCollectionParser(f, sigmaconfigs, rulefilter, sigmafile) results = parser.generate(backend) - + nb_result = len(list(copy.deepcopy(results))) inc_filenane = None if nb_result < 2 else 0 - newline_separator = '\0' if cmdargs.print0 else '\n' + + results = list(results) # Since results is an iterator and used twice we convert it a list for result in results: if not fileprefix == None and not inc_filenane == None: #yml action try: @@ -273,8 +297,22 @@ def main(): except (IOError, OSError) as e: print("Failed to open output file '%s': %s" % (filename, str(e)), file=sys.stderr) exit(ERR_OUTPUT) - print(result, file=out, end=newline_separator) - + if not cmdargs.output_fields: + print(result, file=out, end=newline_separator) + + if cmdargs.output_fields: # Handle output fields + output={} + f.seek(0) + docs = yaml.load_all(f, Loader=yaml.FullLoader) + for doc in docs: + for k,v in doc.items(): + if k in output_fields_filtered: + output[k] = v + output['rule'] = [result for result in results] + if "filename" in output_fields_filtered: + output['filename'] = str(sigmafile.name) + output_array.append(output) + if nb_result == 0: # elastalert return "results=[]" so get a error with out not def if not fileprefix == None and not inc_filenane == None: #yml action try: @@ -293,7 +331,7 @@ def main(): except (IOError, OSError) as e: print("Failed to open output file '%s': %s" % (filename, str(e)), file=sys.stderr) exit(ERR_OUTPUT) - + except OSError as e: print("Failed to open Sigma file %s: %s" % (sigmafile, str(e)), file=sys.stderr) error = ERR_OPEN_SIGMA_RULE @@ -321,7 +359,6 @@ def main(): sys.exit(error) except (NotImplementedError, TypeError) as e: print("An unsupported feature is required for this Sigma rule (%s): " % (sigmafile) + str(e), file=sys.stderr) - print("Feel free to contribute for fun and fame, this is open source :) -> https://github.com/Neo23x0/sigma", file=sys.stderr) if not cmdargs.ignore_backend_errors: error = ERR_NOT_IMPLEMENTED if not cmdargs.defer_abort: @@ -343,10 +380,20 @@ def main(): f.close() except: pass - + result = backend.finalize() if result: print(result, file=out) + + if cmdargs.output_fields: + if cmdargs.output_format == 'json': + print(json.dumps(output_array, indent=4, ensure_ascii=False), file=out) + elif cmdargs.output_format == 'yaml': + yml = ruamel.yaml.YAML() + yml.explicit_start = True + yml.indent(mapping=2, sequence=4, offset=2) + print(ruamel.yaml.round_trip_dump(output_array), file=out) + out.close() sys.exit(error) From cbb03db2ddf098df08bfe30d37e2e0c090d42bb6 Mon Sep 17 00:00:00 2001 From: wagga40 <6437862+wagga40@users.noreply.github.com> Date: Thu, 12 Aug 2021 15:28:45 +0200 Subject: [PATCH 2/3] Fix the way YAML is dumped --- tools/sigma/sigmac.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tools/sigma/sigmac.py b/tools/sigma/sigmac.py index 2e8863c16..77c88641f 100755 --- a/tools/sigma/sigmac.py +++ b/tools/sigma/sigmac.py @@ -389,9 +389,6 @@ def main(): if cmdargs.output_format == 'json': print(json.dumps(output_array, indent=4, ensure_ascii=False), file=out) elif cmdargs.output_format == 'yaml': - yml = ruamel.yaml.YAML() - yml.explicit_start = True - yml.indent(mapping=2, sequence=4, offset=2) print(ruamel.yaml.round_trip_dump(output_array), file=out) out.close() From 13a3e78184d7e626ceb2d837b0a019d850e3b9ee Mon Sep 17 00:00:00 2001 From: wagga40 <6437862+wagga40@users.noreply.github.com> Date: Thu, 12 Aug 2021 15:54:02 +0200 Subject: [PATCH 3/3] Fix options : removed "raw" --- tools/sigma/sigmac.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/sigma/sigmac.py b/tools/sigma/sigmac.py index 77c88641f..556b26d46 100755 --- a/tools/sigma/sigmac.py +++ b/tools/sigma/sigmac.py @@ -113,7 +113,7 @@ def set_argparser(): Select the fields you want by providing their list delimited with commas (no space). Only work with the '--output-format' option and with 'json' or 'yaml' value. available additional fields : title, id, status, description, author, references, fields, falsepositives, level, tags. This option do not have any effect for backends that already format output : elastalert, kibana, splukxml etc. """) - argparser.add_argument("--output-format", "-oF", default="raw", choices=["raw", "json", "yaml"], help="Use only if you want to have JSON or YAML output (default is raw text)") + argparser.add_argument("--output-format", "-oF", choices=["json", "yaml"], help="Use only if you want to have JSON or YAML output (default is raw text)") argparser.add_argument("--output-extention", "-e", default=None, help="Extention of Output file for filename prefix use") argparser.add_argument("--print0", action="store_true", help="Delimit results by NUL-character") argparser.add_argument("--backend-option", "-O", action="append", help="Options and switches that are passed to the backend") @@ -224,7 +224,7 @@ def main(): exit(ERR_CONFIG_PARSING) if cmdargs.output_fields: - if cmdargs.output_format != "raw": + if cmdargs.output_format: output_fields_rejected = [field for field in cmdargs.output_fields.split(",") if field not in allowed_fields] # Not allowed fields if output_fields_rejected: print("These fields are not allowed (check help for allow field list) : %s" % (", ".join(output_fields_rejected)), file=sys.stderr) @@ -232,7 +232,7 @@ def main(): else: output_fields_filtered = [field for field in cmdargs.output_fields.split(",") if field in allowed_fields] # Keep only allowed fields else: - print("The '--output-fields' or '-of' arguments must be use with '--output-format' or '-oF' equal to 'json' or 'yaml'", file=sys.stderr) + print("The '--output-fields' or '-of' arguments must be used with '--output-format' or '-oF' equal to 'json' or 'yaml'", file=sys.stderr) exit(ERR_OUTPUT_FORMAT) backend_options = BackendOptions(cmdargs.backend_option, cmdargs.backend_config)