Merge pull request #172 from splunk/release_v4.1.0

Release v4.1.0

Author: pyth0n1c

.github/workflows/test_against_escu.yml

@@ -0,0 +1,69 @@
+# The default branch of security_content should always be correct.
+# As such, we should use it in our test workflow, here, to ensure
+# that contentctl is also correct and does not throw unexpected errors.
+
+# We should remember that if contentctl introduces NEW validations that have
+# note yet been fixed in security_content, we may see this workflow fail.
+name: test_against_escu
+on:
+  push:
+  pull_request:
+    types: [opened, reopened]
+  schedule:
+    - cron: "44 4 * * *"
+
+jobs:
+  smoketest_escu:
+    strategy:
+      fail-fast: false
+      matrix:
+        python_version: ["3.11", "3.12"]
+        operating_system: ["ubuntu-20.04", "ubuntu-22.04", "macos-latest", "macos-14"]
+        #operating_system: ["ubuntu-20.04", "ubuntu-22.04", "macos-latest"]
+
+
+    runs-on: ${{ matrix.operating_system }}
+    steps:
+      # Checkout the current branch of contentctl repo
+      - name: Checkout repo
+        uses: actions/checkout@v4
+
+      # Checkout the develop (default) branch of security_content
+      - name: Checkout repo
+        uses: actions/checkout@v4
+        with: 
+          path: security_content
+          repository: splunk/security_content
+
+      #Install the given version of Python we will test against
+      - name: Install Required Python Version
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python_version }}
+          architecture: "x64"
+          
+      - name: Install Poetry
+        run: 
+          python -m pip install poetry
+
+      - name: Install contentctl and activate the shell
+        run: |
+          poetry install --no-interaction
+
+
+      - name: Clone the AtomicRedTeam Repo (for extended validation)
+        run: |
+          cd security_content
+          git clone --depth 1 https://github.com/redcanaryco/atomic-red-team
+
+      
+      # We do not separately run validate and build 
+      # since a build ALSO performs a validate
+      - name: Run contentctl build
+        run: |
+          cd security_content
+          poetry run contentctl build --enrichments
+
+      # Do not run a test - it will take far too long!
+      # Do not upload any artifacts
+      

.gitignore

@@ -10,6 +10,7 @@ apps*
 test_results*
 attack_data*
 security_content/
+contentctl.yml
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

contentctl/actions/apav_deploy.py

@@ -61,7 +61,7 @@ def inspectAppAPI(self, config: inspect)->str:
         if not package_path.is_file():
             raise Exception(f"Cannot run Appinspect API on App '{config.app.title}' - "
                             f"no package exists as expected path '{package_path}'.\nAre you "
-                            "trying to 'contentctl acs_deploy' the package BEFORE running 'contentctl build'?")
+                            "trying to 'contentctl deploy_acs' the package BEFORE running 'contentctl build'?")
 
         files = {
             "app_package": open(package_path,"rb"),

contentctl/actions/api_deploy.py

@@ -25,7 +25,8 @@ def buildDetection(self)->dict[str,Any]:
         answers['date'] = datetime.today().strftime('%Y-%m-%d')
         answers['author'] = answers['detection_author']
         del answers['detection_author']
-        answers['data_source'] = answers['data_source']
+        answers['data_sources'] = answers['data_source']
+        del answers['data_source']
         answers['type'] = answers['detection_type']
         del answers['detection_type']
         answers['status'] = "production" #start everything as production since that's what we INTEND the content to become   
@@ -49,6 +50,7 @@ def buildDetection(self)->dict[str,Any]:
         answers['tags']['required_fields'] = ['UPDATE']
         answers['tags']['risk_score'] = 'UPDATE (impact * confidence)/100'
         answers['tags']['security_domain'] = answers['security_domain']
+        del answers["security_domain"]
         answers['tags']['cve'] = ['UPDATE WITH CVE(S) IF APPLICABLE']
 
         #generate the tests section
@@ -64,6 +66,7 @@ def buildDetection(self)->dict[str,Any]:
                 ]
             }
         ]
+        del answers["mitre_attack_ids"]
         return answers
 
     def buildStory(self)->dict[str,Any]:
@@ -111,12 +114,12 @@ def writeObjectNewContent(self, object: dict, subdirectory_name: str, type: NewC
             #make sure the output folder exists for this detection
             output_folder.mkdir(exist_ok=True)
 
-            YmlWriter.writeYmlFile(file_path, object)
+            YmlWriter.writeDetection(file_path, object)
             print("Successfully created detection " + file_path)
 
         elif type == NewContentType.story:
             file_path = os.path.join(self.output_path, 'stories', self.convertNameToFileName(object['name'], object['tags']['product']))
-            YmlWriter.writeYmlFile(file_path, object)
+            YmlWriter.writeStory(file_path, object)
             print("Successfully created story " + file_path)        
 
         else:

contentctl/actions/acs_deploy.py renamed to contentctl/actions/deploy_acs.py

@@ -23,6 +23,7 @@ def execute(self, input_dto: validate) -> DirectorOutputDto:
         director_output_dto = DirectorOutputDto(AtomicTest.getAtomicTestsFromArtRepo(repo_path=input_dto.getAtomicRedTeamRepoPath(), 
                                                                                      enabled=input_dto.enrichments),
                                                 AttackEnrichment.getAttackEnrichment(input_dto),
+                                                CveEnrichment.getCveEnrichment(input_dto),
                                                 [],[],[],[],[],[],[],[],[])