From e52f29dda997cdb3fa084555c0cdcaeee6a73a5d Mon Sep 17 00:00:00 2001
From: Maxime Lamothe-Brassard <maxime@refractionpoint.com>
Date: Wed, 30 Oct 2019 15:23:56 -0500
Subject: [PATCH 01/12] Fix matches operator field set to value instead of re.

---
 tools/sigma/backends/limacharlie.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/tools/sigma/backends/limacharlie.py b/tools/sigma/backends/limacharlie.py
index 3180e2a83..db68f8829 100644
--- a/tools/sigma/backends/limacharlie.py
+++ b/tools/sigma/backends/limacharlie.py
@@ -292,11 +292,15 @@ class LimaCharlieBackend(BaseBackend):
             mappedFiltered = []
             for k in filtered:
                 op, newVal = self._valuePatternToLcOp(k)
-                mappedFiltered.append({
+                newOp = {
                     "op": op,
                     "path": self._fieldMappingInEffect["keywords"],
-                    "value": newVal,
-                })
+                }
+                if op == "matches":
+                    newOp["re"] = newVal
+                else:
+                    newOp["value"] = newVal
+                mappedFiltered.append(newOp)
             filtered = mappedFiltered
         if 1 == len(filtered):
             return filtered[0]

From 102ab3081b19df53ba72d7bfc1336cf738ca8ae6 Mon Sep 17 00:00:00 2001
From: Maxime Lamothe-Brassard <maxime@refractionpoint.com>
Date: Wed, 30 Oct 2019 21:25:14 -0500
Subject: [PATCH 02/12] Fix the convertion from simple wildcard strings to a
 full regular expression so that it is always correct. The previous solution
 just mostly-worked.

---
 tools/sigma/backends/limacharlie.py | 121 ++++++++++++++++++++++------
 1 file changed, 95 insertions(+), 26 deletions(-)

diff --git a/tools/sigma/backends/limacharlie.py b/tools/sigma/backends/limacharlie.py
index db68f8829..53290df21 100644
--- a/tools/sigma/backends/limacharlie.py
+++ b/tools/sigma/backends/limacharlie.py
@@ -344,22 +344,30 @@ class LimaCharlieBackend(BaseBackend):
 
         if isinstance(value, (int, str)):
             op, newVal = self._valuePatternToLcOp(value)
-            return {
+            newOp = {
                 "op": op,
                 "path": fieldname,
-                "value": newVal,
                 "case sensitive": False,
             }
+            if op == "matches":
+                newOp["re"] = newVal
+            else:
+                newOp["value"] = newVal
+            return newOp
         elif isinstance(value, list):
             subOps = []
             for v in value:
                 op, newVal = self._valuePatternToLcOp(v)
-                subOps.append({
+                newOp = {
                     "op": op,
                     "path": fieldname,
-                    "value": newVal,
                     "case sensitive": False,
-                })
+                }
+                if op == "matches":
+                    newOp["re"] = newVal
+                else:
+                    newOp["value"] = newVal
+                subOps.append(newOp)
             if 1 == len(subOps):
                 return subOps[0]
             return {
@@ -395,24 +403,85 @@ class LimaCharlieBackend(BaseBackend):
 
         if not isinstance(val, str):
             return ("is", str(val) if self._isAllStringValues else val)
-        # The following logic is taken from the WDATP backend to translate
-        # the basic wildcard format into proper regular expression.
-        if "*" in val[1:-1]:
-            # Contains a wildcard within, must be translated.
-            # TODO: getting a W605 from the \g escape, this may be broken.
-            val = re.sub('([".^$]|\\\\(?![*?]))', '\\\\\g<1>', val)
-            val = re.sub('\\*', '.*', val)
-            val = re.sub('\\?', '.', val)
-            return ("matches", val)
-        # value possibly only starts and/or ends with *, use prefix/postfix match
-        # TODO: this is actually not correct since the string could end with
-        # a \* expression which would mean it's NOT a wildcard. We'll gloss over
-        # it for now to get something out but it should eventually be fixed
-        # so that it's accurate in all corner cases.
-        if val.endswith("*") and val.startswith("*"):
-            return ("contains", val[1:-1])
-        elif val.endswith("*"):
-            return ("starts with", val[:-1])
-        elif val.startswith("*"):
-            return ("ends with", val[1:])
-        return ("is", val)
+
+        # Is there any wildcard in this string? If not, we can short circuit.
+        if "*" not in val and "?" not in val:
+            return ("is", val)
+
+        # Now we do a small optimization for the shortcut operators
+        # available in LC.
+        isStartsWithWildcard = False
+        isEndsWithWildcard = False
+        tmpVal = val
+        if tmpVal.startswith("*"):
+            isStartsWithWildcard = True
+            tmpVal = tmpVal[1:]
+        if tmpVal.endswith("*") and not tmpVal.endswith("\\*"):
+            isEndsWithWildcard = True
+            tmpVal = tmpVal[:-1]
+
+        # Check to see if there are any other wildcards. If there are
+        # we cannot use our shortcuts.
+        if "*" not in tmpVal and "?" not in tmpVal:
+            if isStartsWithWildcard and isEndsWithWildcard:
+                return ("contains", tmpVal)
+
+            if isStartsWithWildcard:
+                return ("ends with", tmpVal)
+
+            if isEndsWithWildcard:
+                return ("starts with", tmpVal)
+
+        # This is messy, but it is accurate in generating a RE based on
+        # the simplified wildcard system, while also supporting the
+        # escaping of those wildcards.
+        segments = []
+        tmpVal = val
+        while True:
+            nEscapes = 0
+            for i in range(len(tmpVal)):
+                # We keep a running count of backslash escape
+                # characters we see so that if we meet a wildcard
+                # we can tell whether the wildcard is escaped
+                # (with odd number of escapes) or if it's just a
+                # backslash literal before a wildcard (even number).
+                if "\\" == tmpVal[i]:
+                    nEscapes += 1
+                    continue
+
+                if "*" == tmpVal[i]:
+                    if 0 == nEscapes:
+                        segments.append(re.escape(tmpVal[:i]))
+                        segments.append(".*")
+                    elif nEscapes % 2 == 0:
+                        segments.append(re.escape(tmpVal[:i - nEscapes]))
+                        segments.append(tmpVal[i - nEscapes:i])
+                        segments.append(".*")
+                    else:
+                        segments.append(re.escape(tmpVal[:i - nEscapes]))
+                        segments.append(tmpVal[i - nEscapes:i + 1])
+                    tmpVal = tmpVal[i + 1:]
+                    break
+
+                if "?" == tmpVal[i]:
+                    if 0 == nEscapes:
+                        segments.append(re.escape(tmpVal[:i]))
+                        segments.append(".")
+                    elif nEscapes % 2 == 0:
+                        segments.append(re.escape(tmpVal[:i - nEscapes]))
+                        segments.append(tmpVal[i - nEscapes:i])
+                        segments.append(".")
+                    else:
+                        segments.append(re.escape(tmpVal[:i - nEscapes]))
+                        segments.append(tmpVal[i - nEscapes:i + 1])
+                    tmpVal = tmpVal[i + 1:]
+                    break
+
+                nEscapes = 0
+            else:
+                segments.append(re.escape(tmpVal))
+                break
+
+        val = ''.join(segments)
+
+        return ("matches", val)
\ No newline at end of file

From 9aedb8f76410e71e64f73e6fef7e114acf637f43 Mon Sep 17 00:00:00 2001
From: Maxime Lamothe-Brassard <maxime@refractionpoint.com>
Date: Wed, 30 Oct 2019 21:34:29 -0500
Subject: [PATCH 03/12] Adding another exception case to get more "contains"
 shortcuts instead of REs.

---
 tools/sigma/backends/limacharlie.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/sigma/backends/limacharlie.py b/tools/sigma/backends/limacharlie.py
index 53290df21..54c8ff7d5 100644
--- a/tools/sigma/backends/limacharlie.py
+++ b/tools/sigma/backends/limacharlie.py
@@ -416,7 +416,7 @@ class LimaCharlieBackend(BaseBackend):
         if tmpVal.startswith("*"):
             isStartsWithWildcard = True
             tmpVal = tmpVal[1:]
-        if tmpVal.endswith("*") and not tmpVal.endswith("\\*"):
+        if tmpVal.endswith("*") and not (tmpVal.endswith("\\*") and not tmpVal.endswith("\\\\*")):
             isEndsWithWildcard = True
             tmpVal = tmpVal[:-1]
 

From 0b9a3f3a086bf4c356a4b2704b331fc5ab8fd5ab Mon Sep 17 00:00:00 2001
From: Maxime Lamothe-Brassard <maxime@refractionpoint.com>
Date: Thu, 31 Oct 2019 11:15:07 -0500
Subject: [PATCH 04/12] Refactor to better support keyword fields.

---
 tools/sigma/backends/limacharlie.py | 82 ++++++++++++++++++-----------
 1 file changed, 50 insertions(+), 32 deletions(-)

diff --git a/tools/sigma/backends/limacharlie.py b/tools/sigma/backends/limacharlie.py
index 54c8ff7d5..df98ad334 100644
--- a/tools/sigma/backends/limacharlie.py
+++ b/tools/sigma/backends/limacharlie.py
@@ -42,7 +42,7 @@ SigmaLCConfig = namedtuple('SigmaLCConfig', [
     'preConditions',
     'fieldMappings',
     'isAllStringValues',
-    'isKeywordsSupported',
+    'keywordField',
 ])
 _allFieldMappings = {
     "windows/process_creation/": SigmaLCConfig(
@@ -71,7 +71,7 @@ _allFieldMappings = {
             "Command": "event/COMMAND_LINE",
         },
         isAllStringValues = False,
-        isKeywordsSupported = False
+        keywordField = "event/COMMAND_LINE"
     ),
     "windows//": SigmaLCConfig(
         topLevelParams = {
@@ -81,7 +81,7 @@ _allFieldMappings = {
         preConditions = None,
         fieldMappings = _windowsEventLogFieldName,
         isAllStringValues = True,
-        isKeywordsSupported = False
+        keywordField = None
     ),
     "windows_defender//": SigmaLCConfig(
         topLevelParams = {
@@ -91,7 +91,7 @@ _allFieldMappings = {
         preConditions = None,
         fieldMappings = _windowsEventLogFieldName,
         isAllStringValues = True,
-        isKeywordsSupported = False
+        keywordField = None
     ),
     "dns//": SigmaLCConfig(
         topLevelParams = {
@@ -102,7 +102,7 @@ _allFieldMappings = {
             "query": "event/DOMAIN_NAME",
         },
         isAllStringValues = False,
-        isKeywordsSupported = False
+        keywordField = None
     ),
     "linux//": SigmaLCConfig(
         topLevelParams = {
@@ -115,12 +115,12 @@ _allFieldMappings = {
             "op": "is linux",
         },
         fieldMappings = {
-            "keywords": "event/COMMAND_LINE",
             "exe": "event/FILE_PATH",
             "type": None,
         },
         isAllStringValues = False,
-        isKeywordsSupported = True),
+        keywordField = 'event/COMMAND_LINE'
+    ),
     "unix//": SigmaLCConfig(
         topLevelParams = {
             "events": [
@@ -132,12 +132,12 @@ _allFieldMappings = {
             "op": "is linux",
         },
         fieldMappings = {
-            "keywords": "event/COMMAND_LINE",
             "exe": "event/FILE_PATH",
             "type": None,
         },
         isAllStringValues = False,
-        isKeywordsSupported = True),
+        keywordField = 'event/COMMAND_LINE'
+    ),
     "netflow//": SigmaLCConfig(
         topLevelParams = {
             "event": "NETWORK_CONNECTIONS",
@@ -148,7 +148,8 @@ _allFieldMappings = {
             "source.port": "event/NETWORK_ACTIVITY/SOURCE/PORT",
         },
         isAllStringValues = False,
-        isKeywordsSupported = True)
+        keywordField = None
+    ),
 }
 
 class LimaCharlieBackend(BaseBackend):
@@ -183,7 +184,7 @@ class LimaCharlieBackend(BaseBackend):
 
         # See if we have a definition for the source combination.
         mappingKey = "%s/%s/%s" % (product, category, service)
-        topFilter, preCond, mappings, isAllStringValues, isKeywordsSupported = _allFieldMappings.get(mappingKey, tuple([None, None, None, None, None]))
+        topFilter, preCond, mappings, isAllStringValues, keywordField = _allFieldMappings.get(mappingKey, tuple([None, None, None, None, None]))
         if mappings is None:
             raise NotImplementedError("Log source %s/%s/%s not supported by backend." % (product, category, service))
 
@@ -197,7 +198,7 @@ class LimaCharlieBackend(BaseBackend):
         self._isAllStringValues = isAllStringValues
 
         # Are we supporting keywords full text search?
-        self._isKeywordsSupported = isKeywordsSupported
+        self._keywordField = keywordField
 
         # Call the original generation code.
         detectComponent = super().generate(sigmaparser)
@@ -256,6 +257,7 @@ class LimaCharlieBackend(BaseBackend):
         # and only convert to string (yaml) once the
         # whole thing is assembled.
         result = self.generateNode(parsed.parsedSearch)
+
         if self._preCondition is not None:
             result = {
                 "op": "and",
@@ -271,6 +273,10 @@ class LimaCharlieBackend(BaseBackend):
         filtered = [ g for g in generated if g is not None ]
         if not filtered:
             return None
+
+        # Map any possible keywords.
+        filtered = self._mapKeywordVals(filtered)
+
         if 1 == len(filtered):
             return filtered[0]
         return {
@@ -283,25 +289,10 @@ class LimaCharlieBackend(BaseBackend):
         filtered = [g for g in generated if g is not None]
         if not filtered:
             return None
-        if isinstance(filtered[0], str):
-            if not self._isKeywordsSupported:
-                raise NotImplementedError("Full-text keyboard searches not supported.")
-            # This seems to be indicative only of "keywords" which are mostly
-            # representative of full-text searches. We don't suport that but
-            # in some data sources we can alias them to an actual field.
-            mappedFiltered = []
-            for k in filtered:
-                op, newVal = self._valuePatternToLcOp(k)
-                newOp = {
-                    "op": op,
-                    "path": self._fieldMappingInEffect["keywords"],
-                }
-                if op == "matches":
-                    newOp["re"] = newVal
-                else:
-                    newOp["value"] = newVal
-                mappedFiltered.append(newOp)
-            filtered = mappedFiltered
+
+        # Map any possible keywords.
+        filtered = self._mapKeywordVals(filtered)
+
         if 1 == len(filtered):
             return filtered[0]
         return {
@@ -484,4 +475,31 @@ class LimaCharlieBackend(BaseBackend):
 
         val = ''.join(segments)
 
-        return ("matches", val)
\ No newline at end of file
+        return ("matches", val)
+
+    def _mapKeywordVals(self, values):
+        mapped = []
+
+        for val in values:
+            if not isinstance(val, str):
+                mapped.append(val)
+                continue
+
+            if self._keywordField is None:
+                raise NotImplementedError("Full-text keyboard searches not supported.")
+
+            # This seems to be indicative only of "keywords" which are mostly
+            # representative of full-text searches. We don't suport that but
+            # in some data sources we can alias them to an actual field.
+            op, newVal = self._valuePatternToLcOp(val)
+            newOp = {
+                "op": op,
+                "path": self._keywordField,
+            }
+            if op == "matches":
+                newOp["re"] = newVal
+            else:
+                newOp["value"] = newVal
+            mapped.append(newOp)
+
+        return mapped
\ No newline at end of file

From 6f2f1d2bd725abf8b6b754fd9c9a3a8c11b42ae6 Mon Sep 17 00:00:00 2001
From: Maxime Lamothe-Brassard <maxime@refractionpoint.com>
Date: Thu, 31 Oct 2019 13:40:41 -0500
Subject: [PATCH 05/12] Add ability to map fields and values based on
 callbacks.

---
 tools/sigma/backends/limacharlie.py | 28 +++++++++++++++++++++++++---
 1 file changed, 25 insertions(+), 3 deletions(-)

diff --git a/tools/sigma/backends/limacharlie.py b/tools/sigma/backends/limacharlie.py
index df98ad334..2ad9a8aa4 100644
--- a/tools/sigma/backends/limacharlie.py
+++ b/tools/sigma/backends/limacharlie.py
@@ -35,8 +35,9 @@ def _windowsEventLogFieldName(fieldName):
 # - top-level parameters
 # - pre-condition is a D&R rule node filtering relevant events.
 # - field mappings is a dict with a mapping or a callable to convert the field name.
+#       Individual mapping values can also be callabled(fieldname, value) returning a new fieldname and value.
 # - isAllStringValues is a bool indicating whether all values should be converted to string.
-# - isKeywordsSupported is a bool indicating if full-text keyword searches are supported.
+# - keywordField is the field name to alias for keywords if supported or None if not.
 SigmaLCConfig = namedtuple('SigmaLCConfig', [
     'topLevelParams',
     'preConditions',
@@ -63,7 +64,7 @@ _allFieldMappings = {
             "User": "event/USER_NAME",
             # This field is redundant in LC, it seems to always be used with Image
             # so we will ignore it.
-            "OriginalFileName": None,
+            "OriginalFileName": lambda fn, fv: ("event/FILE_PATH", "*" + fv),
             # Custom field names coming from somewhere unknown.
             "NewProcessName": "event/FILE_PATH",
             "ProcessCommandLine": "event/COMMAND_LINE",
@@ -318,14 +319,22 @@ class LimaCharlieBackend(BaseBackend):
     def generateMapItemNode(self, node):
         fieldname, value = node
 
+        fieldNameAndValCallback = None
+
         # The mapping can be a dictionary of mapping or a callable
         # to get the correct value.
         if callable(self._fieldMappingInEffect):
             fieldname = self._fieldMappingInEffect(fieldname)
         else:
             try:
-                fieldname = self._fieldMappingInEffect[fieldname]
+                # The mapping can also be a callable that will
+                # return a mapped key AND value.
+                if callable(self._fieldMappingInEffect[fieldname]):
+                    fieldNameAndValCallback = self._fieldMappingInEffect[fieldname]
+                else:
+                    fieldname = self._fieldMappingInEffect[fieldname]
             except:
+                raise
                 raise NotImplementedError("Field name %s not supported by backend." % (fieldname,))
 
         # If fieldname returned is None, it's a special case where we
@@ -334,6 +343,8 @@ class LimaCharlieBackend(BaseBackend):
             return None
 
         if isinstance(value, (int, str)):
+            if fieldNameAndValCallback is not None:
+                fieldname, value = fieldNameAndValCallback(fieldname, value)
             op, newVal = self._valuePatternToLcOp(value)
             newOp = {
                 "op": op,
@@ -348,6 +359,8 @@ class LimaCharlieBackend(BaseBackend):
         elif isinstance(value, list):
             subOps = []
             for v in value:
+                if fieldNameAndValCallback is not None:
+                    fieldname, v = fieldNameAndValCallback(fieldname, v)
                 op, newVal = self._valuePatternToLcOp(v)
                 newOp = {
                     "op": op,
@@ -367,6 +380,8 @@ class LimaCharlieBackend(BaseBackend):
             }
         elif isinstance(value, SigmaTypeModifier):
             if isinstance(value, SigmaRegularExpressionModifier):
+                if fieldNameAndValCallback is not None:
+                    fieldname, value = fieldNameAndValCallback(fieldname, value)
                 return {
                     "op": "matches",
                     "path": fieldname,
@@ -375,6 +390,8 @@ class LimaCharlieBackend(BaseBackend):
             else:
                 raise TypeError("Backend does not support TypeModifier: %s" % (str(type(value))))
         elif value is None:
+            if fieldNameAndValCallback is not None:
+                fieldname, value = fieldNameAndValCallback(fieldname, value)
             return {
                 "op": "exists",
                 "not": True,
@@ -478,6 +495,11 @@ class LimaCharlieBackend(BaseBackend):
         return ("matches", val)
 
     def _mapKeywordVals(self, values):
+        # This function ensures that the list of values passed
+        # are proper D&R operations, if they are strings it indicates
+        # they were requested as keyword matches. We only support
+        # keyword matches when specific in the config where we just
+        # map them to the most common field in LC that makes sense.
         mapped = []
 
         for val in values:

From 0c6b9e532be1c46a3f72f6c20f7570a2587ba180 Mon Sep 17 00:00:00 2001
From: Maxime Lamothe-Brassard <maxime@refractionpoint.com>
Date: Thu, 31 Oct 2019 13:45:38 -0500
Subject: [PATCH 06/12] Remove debugging statement

---
 tools/sigma/backends/limacharlie.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tools/sigma/backends/limacharlie.py b/tools/sigma/backends/limacharlie.py
index 2ad9a8aa4..74037cc6c 100644
--- a/tools/sigma/backends/limacharlie.py
+++ b/tools/sigma/backends/limacharlie.py
@@ -334,7 +334,6 @@ class LimaCharlieBackend(BaseBackend):
                 else:
                     fieldname = self._fieldMappingInEffect[fieldname]
             except:
-                raise
                 raise NotImplementedError("Field name %s not supported by backend." % (fieldname,))
 
         # If fieldname returned is None, it's a special case where we

From c2e621cf08007da716701c60f8b4d52ad8a047ae Mon Sep 17 00:00:00 2001
From: Maxime Lamothe-Brassard <maxime@refractionpoint.com>
Date: Thu, 31 Oct 2019 15:29:31 -0500
Subject: [PATCH 07/12] Fixing another edge case with string escape.

---
 tools/sigma/backends/limacharlie.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/tools/sigma/backends/limacharlie.py b/tools/sigma/backends/limacharlie.py
index 74037cc6c..884d502f4 100644
--- a/tools/sigma/backends/limacharlie.py
+++ b/tools/sigma/backends/limacharlie.py
@@ -425,7 +425,13 @@ class LimaCharlieBackend(BaseBackend):
             tmpVal = tmpVal[1:]
         if tmpVal.endswith("*") and not (tmpVal.endswith("\\*") and not tmpVal.endswith("\\\\*")):
             isEndsWithWildcard = True
-            tmpVal = tmpVal[:-1]
+            if tmpVal.endswith("\\\\*"):
+                # An extra \ had to be there so it didn't escapte the
+                # *, but since we plan on removing the *, we can also
+                # remove one \.
+                tmpVal = tmpVal[:-2]
+            else:
+                tmpVal = tmpVal[:-1]
 
         # Check to see if there are any other wildcards. If there are
         # we cannot use our shortcuts.

From b7018bcd4ad0cb1e1b4dc5b2ff8e4750f4c4c9b8 Mon Sep 17 00:00:00 2001
From: Maxime Lamothe-Brassard <maxime@refractionpoint.com>
Date: Fri, 1 Nov 2019 11:47:53 -0500
Subject: [PATCH 08/12] Adding a post-mapper mechanism to fix some common
 issues in Sigma rules to LC.

---
 tools/sigma/backends/limacharlie.py | 77 ++++++++++++++++++++++++-----
 1 file changed, 64 insertions(+), 13 deletions(-)

diff --git a/tools/sigma/backends/limacharlie.py b/tools/sigma/backends/limacharlie.py
index 884d502f4..65956d957 100644
--- a/tools/sigma/backends/limacharlie.py
+++ b/tools/sigma/backends/limacharlie.py
@@ -28,6 +28,23 @@ def _windowsEventLogFieldName(fieldName):
         return 'Event/System/EventID'
     return 'Event/EventData/%s' % (fieldName,)
 
+def _mapProcessCreationOperations(node):
+    # Here we fix some common pitfalls found in rules
+    # in a consistent fashion (already process to D&R rule).
+
+    # First fixup is looking for a specific path prefix
+    # based on a specific drive letter. There are many cases
+    # where the driver letter can change or where the early
+    # boot process refers to it as "\Device\HarddiskVolume1\".
+    if ("starts with" == node["op"] and
+        "event/FILE_PATH" == node["path"] and
+        node["value"].lower().startswith("c:\\")):
+        node["op"] = "matches"
+        node["re"] = "^(?:(?:.:)|(?:\\\\Device\\\\HarddiskVolume.))\\\\%s" % (re.escape(node["value"][3:]),)
+        del(node["value"])
+
+    return node
+
 # We support many different log sources so we keep different mapping depending
 # on the log source and category.
 # The mapping key is product/category/service.
@@ -38,12 +55,14 @@ def _windowsEventLogFieldName(fieldName):
 #       Individual mapping values can also be callabled(fieldname, value) returning a new fieldname and value.
 # - isAllStringValues is a bool indicating whether all values should be converted to string.
 # - keywordField is the field name to alias for keywords if supported or None if not.
+# - postOpMapper is a callback that can modify an operation once it has been generated.
 SigmaLCConfig = namedtuple('SigmaLCConfig', [
     'topLevelParams',
     'preConditions',
     'fieldMappings',
     'isAllStringValues',
     'keywordField',
+    'postOpMapper',
 ])
 _allFieldMappings = {
     "windows/process_creation/": SigmaLCConfig(
@@ -72,7 +91,8 @@ _allFieldMappings = {
             "Command": "event/COMMAND_LINE",
         },
         isAllStringValues = False,
-        keywordField = "event/COMMAND_LINE"
+        keywordField = "event/COMMAND_LINE",
+        postOpMapper = _mapProcessCreationOperations
     ),
     "windows//": SigmaLCConfig(
         topLevelParams = {
@@ -82,7 +102,8 @@ _allFieldMappings = {
         preConditions = None,
         fieldMappings = _windowsEventLogFieldName,
         isAllStringValues = True,
-        keywordField = None
+        keywordField = None,
+        postOpMapper = None
     ),
     "windows_defender//": SigmaLCConfig(
         topLevelParams = {
@@ -92,7 +113,8 @@ _allFieldMappings = {
         preConditions = None,
         fieldMappings = _windowsEventLogFieldName,
         isAllStringValues = True,
-        keywordField = None
+        keywordField = None,
+        postOpMapper = None
     ),
     "dns//": SigmaLCConfig(
         topLevelParams = {
@@ -103,7 +125,8 @@ _allFieldMappings = {
             "query": "event/DOMAIN_NAME",
         },
         isAllStringValues = False,
-        keywordField = None
+        keywordField = None,
+        postOpMapper = None
     ),
     "linux//": SigmaLCConfig(
         topLevelParams = {
@@ -120,7 +143,8 @@ _allFieldMappings = {
             "type": None,
         },
         isAllStringValues = False,
-        keywordField = 'event/COMMAND_LINE'
+        keywordField = 'event/COMMAND_LINE',
+        postOpMapper = None
     ),
     "unix//": SigmaLCConfig(
         topLevelParams = {
@@ -137,7 +161,8 @@ _allFieldMappings = {
             "type": None,
         },
         isAllStringValues = False,
-        keywordField = 'event/COMMAND_LINE'
+        keywordField = 'event/COMMAND_LINE',
+        postOpMapper = None
     ),
     "netflow//": SigmaLCConfig(
         topLevelParams = {
@@ -149,7 +174,8 @@ _allFieldMappings = {
             "source.port": "event/NETWORK_ACTIVITY/SOURCE/PORT",
         },
         isAllStringValues = False,
-        keywordField = None
+        keywordField = None,
+        postOpMapper = None
     ),
 }
 
@@ -185,7 +211,7 @@ class LimaCharlieBackend(BaseBackend):
 
         # See if we have a definition for the source combination.
         mappingKey = "%s/%s/%s" % (product, category, service)
-        topFilter, preCond, mappings, isAllStringValues, keywordField = _allFieldMappings.get(mappingKey, tuple([None, None, None, None, None]))
+        topFilter, preCond, mappings, isAllStringValues, keywordField, postOpMapper = _allFieldMappings.get(mappingKey, tuple([None, None, None, None, None, None]))
         if mappings is None:
             raise NotImplementedError("Log source %s/%s/%s not supported by backend." % (product, category, service))
 
@@ -201,6 +227,9 @@ class LimaCharlieBackend(BaseBackend):
         # Are we supporting keywords full text search?
         self._keywordField = keywordField
 
+        # Call to fixup all operations after the fact.
+        self._postOpMapper = postOpMapper
+
         # Call the original generation code.
         detectComponent = super().generate(sigmaparser)
 
@@ -267,6 +296,8 @@ class LimaCharlieBackend(BaseBackend):
                     result,
                 ]
             }
+            if self._postOpMapper is not None:
+                result = self._postOpMapper(result)
         return yaml.safe_dump(result)
 
     def generateANDNode(self, node):
@@ -279,11 +310,16 @@ class LimaCharlieBackend(BaseBackend):
         filtered = self._mapKeywordVals(filtered)
 
         if 1 == len(filtered):
+            if self._postOpMapper is not None:
+                filtered[0] = self._postOpMapper(filtered[0])
             return filtered[0]
-        return {
+        result = {
             "op": "and",
             "rules": filtered,
         }
+        if self._postOpMapper is not None:
+            result = self._postOpMapper(result)
+        return result
 
     def generateORNode(self, node):
         generated = [self.generateNode(val) for val in node]
@@ -295,11 +331,16 @@ class LimaCharlieBackend(BaseBackend):
         filtered = self._mapKeywordVals(filtered)
 
         if 1 == len(filtered):
+            if self._postOpMapper is not None:
+                filtered[0] = self._postOpMapper(filtered[0])
             return filtered[0]
-        return {
+        result = {
             "op": "or",
             "rules": filtered,
         }
+        if self._postOpMapper is not None:
+            result = self._postOpMapper(result)
+        return result
 
     def generateNOTNode(self, node):
         generated = self.generateNode(node.item)
@@ -307,7 +348,7 @@ class LimaCharlieBackend(BaseBackend):
             return None
         if not isinstance(generated, dict):
             raise NotImplementedError("Not operator not available on non-dict nodes.")
-        generated['not'] = True
+        generated["not"] = not generated.get("not", False)
         return generated
 
     def generateSubexpressionNode(self, node):
@@ -354,6 +395,8 @@ class LimaCharlieBackend(BaseBackend):
                 newOp["re"] = newVal
             else:
                 newOp["value"] = newVal
+            if self._postOpMapper is not None:
+                newOp = self._postOpMapper(newOp)
             return newOp
         elif isinstance(value, list):
             subOps = []
@@ -370,6 +413,8 @@ class LimaCharlieBackend(BaseBackend):
                     newOp["re"] = newVal
                 else:
                     newOp["value"] = newVal
+                if self._postOpMapper is not None:
+                    newOp = self._postOpMapper(newOp)
                 subOps.append(newOp)
             if 1 == len(subOps):
                 return subOps[0]
@@ -381,21 +426,27 @@ class LimaCharlieBackend(BaseBackend):
             if isinstance(value, SigmaRegularExpressionModifier):
                 if fieldNameAndValCallback is not None:
                     fieldname, value = fieldNameAndValCallback(fieldname, value)
-                return {
+                result = {
                     "op": "matches",
                     "path": fieldname,
                     "re": re.compile(value),
                 }
+                if self._postOpMapper is not None:
+                    result = self._postOpMapper(result)
+                return result
             else:
                 raise TypeError("Backend does not support TypeModifier: %s" % (str(type(value))))
         elif value is None:
             if fieldNameAndValCallback is not None:
                 fieldname, value = fieldNameAndValCallback(fieldname, value)
-            return {
+            result = {
                 "op": "exists",
                 "not": True,
                 "path": fieldname,
             }
+            if self._postOpMapper is not None:
+                result = self._postOpMapper(result)
+            return result
         else:
             raise TypeError("Backend does not support map values of type " + str(type(value)))
 

From 1b9054c1f3ee9a240a8e6b6b46864ebbe2e304df Mon Sep 17 00:00:00 2001
From: Maxime Lamothe-Brassard <maxime@refractionpoint.com>
Date: Tue, 5 Nov 2019 08:33:21 -0500
Subject: [PATCH 09/12] Adding some comments

---
 tools/sigma/backends/limacharlie.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/tools/sigma/backends/limacharlie.py b/tools/sigma/backends/limacharlie.py
index 65956d957..d5682d492 100644
--- a/tools/sigma/backends/limacharlie.py
+++ b/tools/sigma/backends/limacharlie.py
@@ -30,7 +30,7 @@ def _windowsEventLogFieldName(fieldName):
 
 def _mapProcessCreationOperations(node):
     # Here we fix some common pitfalls found in rules
-    # in a consistent fashion (already process to D&R rule).
+    # in a consistent fashion (already processed to D&R rule).
 
     # First fixup is looking for a specific path prefix
     # based on a specific drive letter. There are many cases
@@ -459,6 +459,7 @@ class LimaCharlieBackend(BaseBackend):
         # or into altered values to be functionally equivalent using
         # a few different LC D&R rule operators.
 
+        # No point evaluating non-strings.
         if not isinstance(val, str):
             return ("is", str(val) if self._isAllStringValues else val)
 
@@ -467,7 +468,9 @@ class LimaCharlieBackend(BaseBackend):
             return ("is", val)
 
         # Now we do a small optimization for the shortcut operators
-        # available in LC.
+        # available in LC. We try to see if the wildcards are around
+        # the main value, but NOT within. If that's the case we can
+        # use the "starts with", "ends with" or "contains" operators.
         isStartsWithWildcard = False
         isEndsWithWildcard = False
         tmpVal = val
@@ -554,11 +557,12 @@ class LimaCharlieBackend(BaseBackend):
         # This function ensures that the list of values passed
         # are proper D&R operations, if they are strings it indicates
         # they were requested as keyword matches. We only support
-        # keyword matches when specific in the config where we just
+        # keyword matches when specified in the config. We generally just
         # map them to the most common field in LC that makes sense.
         mapped = []
 
         for val in values:
+            # Non-keywords are just passed through.
             if not isinstance(val, str):
                 mapped.append(val)
                 continue

From ef14ee542d6839da8d7920565293e3427b942c36 Mon Sep 17 00:00:00 2001
From: Thomas Patzke <thomas@patzke.org>
Date: Tue, 5 Nov 2019 23:04:13 +0100
Subject: [PATCH 10/12] Added modifiers: startswith and endswith

---
 tests/test-modifiers.yml                  |  2 ++
 tools/sigma/parser/modifiers/transform.py | 20 ++++++++++++++++++++
 2 files changed, 22 insertions(+)

diff --git a/tests/test-modifiers.yml b/tests/test-modifiers.yml
index e856e19d5..8e578234c 100644
--- a/tests/test-modifiers.yml
+++ b/tests/test-modifiers.yml
@@ -13,4 +13,6 @@ detection:
             - foo
             - bar
             - bla
+        end|endswith: test
+        start|startswith: test
     condition: selection
diff --git a/tools/sigma/parser/modifiers/transform.py b/tools/sigma/parser/modifiers/transform.py
index 63b36fd8c..c30f92daf 100644
--- a/tools/sigma/parser/modifiers/transform.py
+++ b/tools/sigma/parser/modifiers/transform.py
@@ -31,6 +31,26 @@ class SigmaContainsModifier(ListOrStringModifierMixin, SigmaTransformModifier):
             val += "*"
         return val
 
+class SigmaStartswithModifier(ListOrStringModifierMixin, SigmaTransformModifier):
+    """Add *-wildcard before and after all string(s)"""
+    identifier = "startswith"
+    active = True
+
+    def apply_str(self, val : str):
+        if not val.endswith("*"):
+            val += "*"
+        return val
+
+class SigmaEndswithModifier(ListOrStringModifierMixin, SigmaTransformModifier):
+    """Add *-wildcard before and after all string(s)"""
+    identifier = "endswith"
+    active = True
+
+    def apply_str(self, val : str):
+        if not val.startswith("*"):
+            val = "*" + val
+        return val
+
 class SigmaAllValuesModifier(SigmaTransformModifier):
     """Override default OR-linking behavior for list with AND-linking of all list values"""
     identifier = "all"

From 8b7560c2f4946581401e28e8fdfc4ca25c5676e0 Mon Sep 17 00:00:00 2001
From: Thomas Patzke <thomas@patzke.org>
Date: Thu, 7 Nov 2019 23:08:44 +0100
Subject: [PATCH 11/12] Added changelog

---
 CHANGELOG.md    | 89 +++++++++++++++++++++++++++++++++++++++++++++++++
 CHANGELOG.md.j2 | 38 +++++++++++++++++++++
 2 files changed, 127 insertions(+)
 create mode 100644 CHANGELOG.md
 create mode 100644 CHANGELOG.md.j2

diff --git a/CHANGELOG.md b/CHANGELOG.md
new file mode 100644
index 000000000..cf82e1c2f
--- /dev/null
+++ b/CHANGELOG.md
@@ -0,0 +1,89 @@
+# Release Notes
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html)
+from version 0.14.0.
+
+## Unreleased
+
+Changes from this section will be contained in the next release.
+
+### Added
+
+* sigma-similarity tool
+* LimaCharlie backend
+* Default configurations for some backends that are used if no configuration is passed.
+* Value modifiers:
+    * startswith
+    * endswith
+
+### Changed
+
+* Removal of line breaks in elastalert output
+* Searches not bound to fields are restricted to keyword fields in es-qs backend
+* Graylog backend now based on es-qs backend
+
+## 0.13
+
+### Added
+
+* Index mappings for Sumologic
+* Malicious cmdlets in wdatp
+* QRadar support for keyword searches
+* QRadar mapping improvements
+* QRadar field selection
+* QRadar type regex modifier support
+* Elasticsearch keyword field blacklisting with wildcards
+* Added dateField configuration parameter in xpack-watcher backend
+* Field mappings in configurations
+* Field name mapping for conditional fields
+* Value modifiers:
+    * utf16
+    * utf16le
+    * wide
+    * utf16be
+
+### Changed
+
+* Improved --backend-config help text
+
+### Fixed
+
+* Backend errors in ala
+* Slash escaping within es-dsl wildcard queries
+* QRadar backend config
+* QRadar field name and value escaping and handling
+* Elasticsearch wildcard detection pattern
+* Aggregation on keyword field in es-dsl backend
+
+## 0.12.1
+
+### Fixed
+
+* Missing build dependency
+
+## 0.12
+
+### Added
+
+* Usage of "Channel" field in ELK Windows configuration
+* Fields to mappings
+* xpack-watcher actions index and webhook
+* Config for Winlogbeat 7.x
+* Value modifiers
+* Regular expression support
+
+### Changed
+
+* Warning/error messages
+* Sumologic value cleaning
+* Explicit OR for Elasticsearch query strings
+* Listing of available configurations on missing configuration error
+
+### Fixed
+
+* Conditions in es-dsl backend
+* Sumologic handling of null values
+* Ignore timeframe detection keyword in all/any of conditions
diff --git a/CHANGELOG.md.j2 b/CHANGELOG.md.j2
new file mode 100644
index 000000000..8dd07eee2
--- /dev/null
+++ b/CHANGELOG.md.j2
@@ -0,0 +1,38 @@
+## {{ version.minor }}.{{ version.major }}.{{ version.patch }} ({{ date }})
+
+### Added
+
+{% for item in added %}
+* {{ item | indent(2) }}
+{% endfor %}
+
+### Changed
+
+{% for item in changed %}
+* {{ item | indent(2) }}
+{% endfor %}
+
+### Deprecated
+
+{% for item in deprecated %}
+* {{ item | indent(2) }}
+{% endfor %}
+
+### Removed
+
+{% for item in removed %}
+* {{ item | indent(2) }}
+{% endfor %}
+
+### Fixed
+
+{% for item in fixed %}
+* {{ item | indent(2) }}
+{% endfor %}
+
+### Security
+
+{% for item in security %}
+* {{ item | indent(2) }}
+{% endfor %}
+

From 465e41bfbbc6da173e726a162e34cbc6203bcfc3 Mon Sep 17 00:00:00 2001
From: Thomas Patzke <thomas@patzke.org>
Date: Fri, 8 Nov 2019 22:31:02 +0100
Subject: [PATCH 12/12] Added regular expression support in es-dsl backend

---
 CHANGELOG.md                          | 1 +
 tools/sigma/backends/elasticsearch.py | 9 ++++++---
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index cf82e1c2f..c7e41c48a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -15,6 +15,7 @@ Changes from this section will be contained in the next release.
 * sigma-similarity tool
 * LimaCharlie backend
 * Default configurations for some backends that are used if no configuration is passed.
+* Regular expression support for es-dsl backend (propagates to backends derived from this like elastalert-dsl)
 * Value modifiers:
     * startswith
     * endswith
diff --git a/tools/sigma/backends/elasticsearch.py b/tools/sigma/backends/elasticsearch.py
index 1a7be9a3d..423b93dc4 100644
--- a/tools/sigma/backends/elasticsearch.py
+++ b/tools/sigma/backends/elasticsearch.py
@@ -212,8 +212,6 @@ class ElasticsearchDSLBackend(RulenameCommentMixin, ElasticsearchWildcardHandlin
 
     def generateMapItemNode(self, node):
         key, value = node
-        if type(value) not in (str, int, list, type(None)):
-            raise TypeError("Map values must be strings, numbers, lists or null, not " + str(type(value)))
         if type(value) is list:
             res = {'bool': {'should': []}}
             for v in value:
@@ -230,7 +228,7 @@ class ElasticsearchDSLBackend(RulenameCommentMixin, ElasticsearchWildcardHandlin
         elif value is None:
             key_mapped = self.fieldNameMapping(key, value)
             return { "bool": { "must_not": { "exists": { "field": key_mapped } } } }
-        else:
+        elif type(value) in (str, int):
             key_mapped = self.fieldNameMapping(key, value)
             if self.matchKeyword:   # searches against keyowrd fields are wildcard searches, phrases otherwise
                 queryType = 'wildcard'
@@ -239,6 +237,11 @@ class ElasticsearchDSLBackend(RulenameCommentMixin, ElasticsearchWildcardHandlin
                 queryType = 'match_phrase'
                 value_cleaned = self.cleanValue(str(value))
             return {queryType: {key_mapped: value_cleaned}}
+        elif isinstance(value, SigmaRegularExpressionModifier):
+            key_mapped = self.fieldNameMapping(key, value)
+            return { 'regexp': { key_mapped: str(value) } }
+        else:
+            raise TypeError("Map values must be strings, numbers, lists, null or regular expression, not " + str(type(value)))
 
     def generateValueNode(self, node):
         return {'multi_match': {'query': node, 'fields': [], 'type': 'phrase'}}