Update ML-DGA docs (#750)

detection_rules/devtools.py

@@ -526,10 +526,16 @@ def validate_ml_detections_asset(directory):
 
     now = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())
 
-    job_paths = list(Path(directory).glob('*.json'))
-    rule_paths = list(Path(directory).glob('*.toml'))
+    all_files = list(Path(directory).glob('*'))
+    job_paths = [f for f in all_files if f.suffix == '.json']
+    rule_paths = [f for f in all_files if f.suffix == '.toml']
+    other_paths = [f for f in Path(directory).glob('*') if f.suffix not in ('.toml', '.json')]
     job_count = len(job_paths)
     rule_count = len(rule_paths)
+    other_count = len(other_paths)
+
+    if 'readme.md' not in [f.name.lower() for f in other_paths]:
+        client_error('Release is missing readme file')
 
     for job in job_paths:
         try:
@@ -562,8 +568,9 @@ def validate_ml_detections_asset(directory):
     click.secho('[!] run `es upload-ml-job` to test jobs on a live stack before releasing', fg='green')
 
     description = {
-        'Experimental ML rules': rule_count,
-        'Experimental ML jobs': str(job_count) + '\n\n----\n\n',
+        'Experimental rules': rule_count,
+        'Experimental ML jobs': job_count,
+        'Other files': str(other_count) + '\n\n----\n\n',
         'DGA release': '<add link to DGA release these detections were built on>',
         'date': now,
         'For details reference': 'https://github.com/elastic/detection-rules/blob/main/docs/ML_DGA.md'

docs/ML_DGA.md

@@ -92,7 +92,8 @@ Any packetbeat documents with the field `dns.question.registered_domain` should
 
 Once packetbeat data is being enriched, there are some rules and ML jobs which can leverage the enriched fields. 
 The experimental rules and jobs will be staged separate from the model bundle under the [releases](https://github.com/elastic/detection-rules/releases) 
-as `ML-experimental-detections-YYYMMDD-N`.
+as `ML-experimental-detections-YYYMMDD-N`. These releases should be considered independent of each other. Any relation
+to previously released experimental detections will be mentioned in the accompanying readme (such as an update to a rule).
 
 Note that if a rule is of `type = "machine_learning"`, then it may be dependent on a uploading and running a machine
 learning job first. If this is the case, it will likely be annotated within the `note` field of the rule.
@@ -167,6 +168,10 @@ All of these checks are automated and can be called with:
 Pay attention to the output to determine any necessary changes. This may not be all inclusive and actual testing on a 
 live stack should always occur even with passing validation before saving to a GitHub release
 
+#### Including a readme for detections release
+
+`ML-experimental-detections-*` releases will need to include a readme to provide an overview of the included files
+
 #### Releasing
 
 Install dependencies with `pip install -r requirements-dev.txt`