Merge branch 'main' into release_v4.2.0

Author: pyth0n1c

.github/workflows/test_against_escu.yml

@@ -53,10 +53,11 @@ jobs:
           poetry install --no-interaction
 
 
-      - name: Clone the AtomicRedTeam Repo (for extended validation)
+      - name: Clone the AtomicRedTeam Repo and the Mitre/CTI repos for testing enrichments  
         run: |
           cd security_content
-          git clone --depth 1 https://github.com/redcanaryco/atomic-red-team
+          git clone --single-branch https://github.com/redcanaryco/atomic-red-team external_repos/atomic-red-team
+          git clone --single-branch https://github.com/mitre/cti external_repos/cti
 
       
       # We do not separately run validate and build 

README.md

@@ -134,7 +134,7 @@ This section is under active development.  It will allow you to a [MITRE Map](ht
 Choose TYPE {detection, story} to create new content for the Content Pack.  The tool will interactively ask a series of questions required for generating a basic piece of content and automatically add it to the Content Pack.
 
 ### contentctl inspect
-This section is under development.  It will enable the user to perform an appinspect of the content pack in preparation for deployment onto a Splunk Instance or via Splunk Cloud.
+This section is under development.  The inspect action performs a number of post-build validations. Primarily, it will enable the user to perform an appinspect of the content pack in preparation for deployment onto a Splunk Instance or via Splunk Cloud. It also compares detections in the new build against a prior build, confirming that any changed detections have had their versions incremented (this comparison happens at the savedsearch.conf level, which is why it must happen after the build). Please also note that new versions of contentctl may result in the generation of different savedsearches.conf files without any content changes in YML (new keys at the .conf level which will necessitate bumping of the version in the YML file).
 
 ### contentctl deploy
 The reason to build content is so that it can be deployed to your environment.  However, deploying content to multiple servers and different types of infrastructure can be tricky and time-consuming.  contentctl makes this easy by supporting a number of different deployment mechanisms. Deployment targets can be defined in [contentctl.yml](/contentctl/templates/contentctl_default.yml).

contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py

@@ -382,12 +382,6 @@ def execute(self):
                 return
 
             try:
-                # NOTE: (THIS CODE HAS MOVED) we handle skipping entire detections differently than
-                #   we do skipping individual test cases; we skip entire detections by excluding
-                #   them to an entirely separate queue, while we skip individual test cases via the
-                #   BaseTest.skip() method, such as when we are skipping all integration tests (see
-                #   DetectionBuilder.skipIntegrationTests)
-                # TODO: are we skipping by production status elsewhere?
                 detection = self.sync_obj.inputQueue.pop()
                 self.sync_obj.currentTestingQueue[self.get_name()] = detection
             except IndexError:

contentctl/actions/initialize.py

@@ -2,8 +2,6 @@
 import shutil
 import os
 import pathlib
-
-from pydantic import RootModel 
 from contentctl.objects.config import test
 from contentctl.output.yml_writer import YmlWriter
 
@@ -17,26 +15,44 @@ def execute(self, config: test) -> None:
 
         YmlWriter.writeYmlFile(str(config.path/'contentctl.yml'), config.model_dump()) 
 
+        
         #Create the following empty directories:
-        for emptyDir in ['lookups', 'baselines', 'docs', 'reporting', 'investigations']:
+        for emptyDir in ['lookups', 'baselines', 'data_sources', 'docs', 'reporting', 'investigations', 
+                            'detections/application', 'detections/cloud', 'detections/endpoint', 
+                            'detections/network', 'detections/web', 'macros', 'stories']:
             #Throw an error if this directory already exists
-            (config.path/emptyDir).mkdir(exist_ok=False)
+            (config.path/emptyDir).mkdir(exist_ok=False, parents=True)
+
+        # If this is not a bare config, then populate
+        # a small amount of content into the directories
+        if not config.bare:
+            #copy the contents of all template directories
+            for templateDir, targetDir in [
+                ('../templates/detections/', 'detections'),
+                ('../templates/data_sources/', 'data_sources'),
+                ('../templates/macros/', 'macros'),
+                ('../templates/stories/', 'stories'),
+            ]:
+                source_directory = pathlib.Path(os.path.dirname(__file__))/templateDir
+                target_directory = config.path/targetDir
+                
+                # Do not throw an exception if the directory exists. In fact, it was 
+                # created above when the structure of the app was created.
+                shutil.copytree(source_directory, target_directory, dirs_exist_ok=True)
 
-
-        #copy the contents of all template directories
+        # The contents of app_template must ALWAYS be copied because it contains
+        # several special files.
+        # For now, we also copy the deployments because the ability to create custom
+        # deployment files is limited with built-in functionality.
         for templateDir, targetDir in [
             ('../templates/app_template/', 'app_template'),
-            ('../templates/deployments/', 'deployments'),
-            ('../templates/detections/', 'detections'),
-            ('../templates/data_sources/', 'data_sources'),
-            ('../templates/macros/','macros'),
-            ('../templates/stories/', 'stories'),
+            ('../templates/deployments/', 'deployments')
         ]:
             source_directory = pathlib.Path(os.path.dirname(__file__))/templateDir
             target_directory = config.path/targetDir
             #Throw an exception if the target exists
             shutil.copytree(source_directory, target_directory, dirs_exist_ok=False)
-        
+
         # Create a README.md file.  Note that this is the README.md for the repository, not the
         # one which will actually be packaged into the app. That is located in the app_template folder.
         shutil.copyfile(pathlib.Path(os.path.dirname(__file__))/'../templates/README.md','README.md')