Merge branch 'main' into enable_acs_deploy

Author: pyth0n1c

.github/workflows/testEndToEnd.yml

@@ -12,7 +12,7 @@ jobs:
       fail-fast: false
       matrix:
         python_version: ["3.11", "3.12"]
-        operating_system: ["ubuntu-20.04", "ubuntu-22.04", "macos-latest", "macos-14"]
+        operating_system: ["ubuntu-20.04", "ubuntu-22.04", "macos-latest", "macos-14", "windows-2022"]
         #operating_system: ["ubuntu-20.04", "ubuntu-22.04", "macos-latest"]
 
 

.github/workflows/test_against_escu.yml

@@ -18,8 +18,10 @@ jobs:
       fail-fast: false
       matrix:
         python_version: ["3.11", "3.12"]
+        
         operating_system: ["ubuntu-20.04", "ubuntu-22.04", "macos-latest", "macos-14"]
-        #operating_system: ["ubuntu-20.04", "ubuntu-22.04", "macos-latest"]
+        # Do not test against ESCU until known character encoding issue is resolved
+        # operating_system: ["ubuntu-20.04", "ubuntu-22.04", "macos-latest", "macos-14", "windows-2022"]
 
 
     runs-on: ${{ matrix.operating_system }}

.vscode/launch.json

@@ -1,5 +1,13 @@
 {
 "configurations": [
+    {
+        "name":"contentctl (pick args)",
+        "type":"debugpy",
+        "request":"launch",
+        "program":"${workspaceFolder}/.venv/bin/contentctl",
+        "console":"integratedTerminal",
+        "cwd":"${env:SECURITY_CONTENT_PATH}",
+        "args":"${command:pickArgs}"},
     {
         "name": "contentctl init",
         "type": "debugpy",

.vscode/settings.json

@@ -3,7 +3,8 @@
     "python.envFile": "${workspaceFolder}/.env",
     "python.testing.cwd": "${workspaceFolder}",
     "python.languageServer": "Pylance",
-    "python.analysis.typeCheckingMode": "strict"
+    "python.analysis.typeCheckingMode": "strict",
+    "editor.defaultFormatter": "ms-python.black-formatter"
 
 
 }

README.md

@@ -3,8 +3,20 @@
 <p align="center">
 <img src="docs/contentctl_logo_white.png" title="In case you're wondering, it's a capybara" alt="contentctl logo" width="250" height="250"></p>
 
-
-
+# contentctl Quick Start Guide
+If you are already familiar with contentctl, the following common commands may be very useful for basic operations
+
+| Operation | Command |
+|-----------|---------|
+| Create a repository | `contentctl init` |
+| Validate Your Content | `contentctl validate` |
+| Validate Your Content, performing MITRE Enrichments | `contentctl validate –-enrichments`|
+| Build Your App | `contentctl build` |
+| Test All the content in your app, pausing so that you can debug a search if it fails | `contentctl test –-post-test-behavior pause_on_failure mode:all` |
+| Test All the content in your app, pausing after every detection to allow debugging | `contentctl test –-post-test-behavior always_pause mode:all` |
+| Test 1 or more specified detections. If you are testing more than one detection, the paths are space-separated. You may also use shell-expanded regexes | `contentctl test –-post-test-behavior always_pause mode:selected --mode.files detections/endpoint/7zip_commandline_to_smb_share_path.yml detections/cloud/aws_multi_factor_authentication_disabled.yml detections/application/okta*` |
+| Diff your current branch with a target_branch and test detections that have been updated. Your current branch **must be DIFFERENT** than the target_branch | `contentctl test –-post-test-behavior always_pause mode:changes –-mode.target_branch develop` |
+| Perform Integration Testing of all content. Note that Enterprise Security MUST be listed as an app in your contentctl.yml folder, otherwise all tests will subsequently fail | `contentctl test –-enable-integration-testing --post-test-behavior never_pause mode:all` |
 
 # Introduction
 #### Security Is Hard 
@@ -65,10 +77,7 @@ Testing is run using [GitHub Hosted Runners](https://docs.github.com/en/actions/
 
 | Requirement | Supported | Description |  Passing Integration Tests |
 | --------------------- | ----- | ---- | ------ |
-| Python <3.9 | No | No support planned.  contentctl tool uses modern language constructs not supported ion Python3.8 and below | N/A |
-| Python 3.9 | Yes | contentctl tool is written in Python | Yes (locally + GitHub Actions) |
-| Python 3.10 | Yes | contentctl tool is written in Python | Yes (locally + GitHub Actions) |
-| Python 3.11 | Yes | contentctl tool is written in Python | Yes (locally + GitHub Actions)  |
+| Python 3.11+ | Yes | contentctl tool is written in Python | Yes (locally + GitHub Actions)  |
 | Docker (local) | Yes | A running Splunk Server is required for Dynamic Testing.  contentctl can automatically create, configure, and destroy this server as a Splunk container during the lifetime of a test. | (locally + GitHub Actions) |
 | Docker (remote) | Planned | A running Splunk Server is required for Dynamic Testing.  contentctl can automatically create, configure, and destroy this server as a Splunk container during the lifetime of a test. | No |
 
@@ -80,7 +89,7 @@ It is typically recommended to install poetry to the Global Python Environment.*
 
 #### Install via pip (recommended): 
 ```
-python3.9 -m venv .venv
+python3.11 -m venv .venv
 source .venv/bin/activate
 pip install contentctl
 ```
@@ -89,7 +98,7 @@ pip install contentctl
 ```
 git clone https://github.com/splunk/contentctl
 cd contentctl
-python3.9 -m pip install poetry
+python3.11 -m pip install poetry
 poetry install
 poetry shell
 contentctl --help

contentctl/actions/build.py

@@ -8,8 +8,9 @@
 from contentctl.input.director import Director, DirectorOutputDto
 from contentctl.output.conf_output import ConfOutput
 from contentctl.output.conf_writer import ConfWriter
-from contentctl.output.ba_yml_output import BAYmlOutput
 from contentctl.output.api_json_output import ApiJsonOutput
+from contentctl.output.data_source_writer import DataSourceWriter
+from contentctl.objects.lookup import Lookup
 import pathlib
 import json
 import datetime
@@ -28,9 +29,20 @@ class Build:
 
 
     def execute(self, input_dto: BuildInputDto) -> DirectorOutputDto:
-        if input_dto.config.build_app:    
+        if input_dto.config.build_app:
+
             updated_conf_files:set[pathlib.Path] = set()
             conf_output = ConfOutput(input_dto.config)
+
+            # Construct a special lookup whose CSV is created at runtime and
+            # written directly into the output folder. It is created with model_construct,
+            # not model_validate, because the CSV does not exist yet.
+            data_sources_lookup_csv_path = input_dto.config.getPackageDirectoryPath() / "lookups" / "data_sources.csv"
+            DataSourceWriter.writeDataSourceCsv(input_dto.director_output_dto.data_sources, data_sources_lookup_csv_path)
+            input_dto.director_output_dto.addContentToDictMappings(Lookup.model_construct(description= "A lookup file that will contain the data source objects for detections.", 
+                                                                        filename=data_sources_lookup_csv_path, 
+                                                                        name="data_sources"))
+
             updated_conf_files.update(conf_output.writeHeaders())
             updated_conf_files.update(conf_output.writeObjects(input_dto.director_output_dto.detections, SecurityContentType.detections))
             updated_conf_files.update(conf_output.writeObjects(input_dto.director_output_dto.stories, SecurityContentType.stories))
@@ -73,17 +85,4 @@ def execute(self, input_dto: BuildInputDto) -> DirectorOutputDto:
 
             print(f"Build of '{input_dto.config.app.title}' API successful to {input_dto.config.getAPIPath()}")
 
-        if input_dto.config.build_ssa:
-            
-            srs_path = input_dto.config.getSSAPath() / 'srs'
-            complex_path = input_dto.config.getSSAPath() / 'complex'
-            shutil.rmtree(srs_path, ignore_errors=True)
-            shutil.rmtree(complex_path, ignore_errors=True)
-            srs_path.mkdir(parents=True)
-            complex_path.mkdir(parents=True)
-            ba_yml_output = BAYmlOutput()
-            ba_yml_output.writeObjects(input_dto.director_output_dto.ssa_detections, str(input_dto.config.getSSAPath()))
-
-            print(f"Build of 'SSA' successful to {input_dto.config.getSSAPath()}")
-                
         return input_dto.director_output_dto

contentctl/actions/convert.py

@@ -76,33 +76,28 @@ def getChanges(self, target_branch:str)->List[Detection]:
                 if diff.delta.status in (DeltaStatus.ADDED, DeltaStatus.MODIFIED, DeltaStatus.RENAMED):
                     #print(f"{DeltaStatus(diff.delta.status).name:<8}:{diff.delta.new_file.raw_path}")
                     decoded_path = pathlib.Path(diff.delta.new_file.raw_path.decode('utf-8'))
-                    if 'app_template/' in str(decoded_path) or 'ssa_detections' in str(decoded_path) or str(self.config.getBuildDir()) in str(decoded_path):
-                        #Ignore anything that is embedded in the app template.
-                        #Also ignore ssa detections
-                        pass
-                    elif 'detections/' in str(decoded_path) and decoded_path.suffix == ".yml":
+                    # Note that we only handle updates to detections, lookups, and macros at this time. All other changes are ignored.
+                    if decoded_path.is_relative_to(self.config.path/"detections") and decoded_path.suffix == ".yml":
                         detectionObject = filepath_to_content_map.get(decoded_path, None)
                         if isinstance(detectionObject, Detection):
                             updated_detections.append(detectionObject)
                         else:
                             raise Exception(f"Error getting detection object for file {str(decoded_path)}")
 
-                    elif 'macros/' in str(decoded_path) and decoded_path.suffix == ".yml":
+                    elif decoded_path.is_relative_to(self.config.path/"macros") and decoded_path.suffix == ".yml":
                         macroObject = filepath_to_content_map.get(decoded_path, None)
                         if isinstance(macroObject, Macro):
                             updated_macros.append(macroObject)
                         else:
                             raise Exception(f"Error getting macro object for file {str(decoded_path)}")
 
-                    elif 'lookups/' in str(decoded_path):
+                    elif decoded_path.is_relative_to(self.config.path/"lookups"):
                         # We need to convert this to a yml. This means we will catch
                         # both changes to a csv AND changes to the YML that uses it
-                        
-                        
                         if decoded_path.suffix == ".yml":
                             updatedLookup = filepath_to_content_map.get(decoded_path, None)
                             if not isinstance(updatedLookup,Lookup):
-                                raise Exception(f"Expected {decoded_path} to be type {type(Lookup)}, but instead if was {(type(lookupObject))}")
+                                raise Exception(f"Expected {decoded_path} to be type {type(Lookup)}, but instead if was {(type(updatedLookup))}")
                             updated_lookups.append(updatedLookup)
 
                         elif decoded_path.suffix == ".csv":
@@ -116,23 +111,27 @@ def getChanges(self, target_branch:str)->List[Detection]:
                                 raise Exception(f"More than 1 Lookup reference the modified CSV file '{decoded_path}': {[l.file_path for l in matched ]}")
                             else:
                                 updatedLookup = matched[0]
+                        elif decoded_path.suffix == ".mlmodel":
+                            # Detected a changed .mlmodel file. However, since we do not have testing for these detections at 
+                            # this time, we will ignore this change.
+                            updatedLookup = None
+                            
+
                         else:
-                            raise Exception(f"Error getting lookup object for file {str(decoded_path)}")
+                            raise Exception(f"Detected a changed file in the lookups/ directory '{str(decoded_path)}'.\n"
+                                            "Only files ending in .csv, .yml, or .mlmodel are supported in this "
+                                            "directory. This file must be removed from the lookups/ directory.")
 
-                        if updatedLookup not in updated_lookups:
-                            # It is possible that both th CSV and YML have been modified for the same lookup,
+                        if updatedLookup is not None and updatedLookup not in updated_lookups:
+                            # It is possible that both the CSV and YML have been modified for the same lookup,
                             # and we do not want to add it twice. 
                             updated_lookups.append(updatedLookup)
 
                     else:
                         pass
                         #print(f"Ignore changes to file {decoded_path} since it is not a detection, macro, or lookup.")
-                
-                # else:
-                #     print(f"{diff.delta.new_file.raw_path}:{DeltaStatus(diff.delta.status).name} (IGNORED)")
-                #     pass
             else:
-                raise Exception(f"Unrecognized type {type(diff)}")
+                raise Exception(f"Unrecognized diff type {type(diff)}")
 
 
         # If a detection has at least one dependency on changed content,
@@ -153,24 +152,25 @@ def getChanges(self, target_branch:str)->List[Detection]:
         #Print out the names of all modified/new content
         modifiedAndNewContentString = "\n - ".join(sorted([d.name for d in updated_detections]))
 
-        print(f"[{len(updated_detections)}] Pieces of modifed and new content to test:\n - {modifiedAndNewContentString}")
+        print(f"[{len(updated_detections)}] Pieces of modifed and new content (this may include experimental/deprecated/manual_test content):\n - {modifiedAndNewContentString}")
         return updated_detections
 
-    def getSelected(self, detectionFilenames:List[FilePath])->List[Detection]:
-        filepath_to_content_map:dict[FilePath, SecurityContentObject] = { obj.file_path:obj for (_,obj) in self.director.name_to_content_map.items() if obj.file_path is not None} 
+    def getSelected(self, detectionFilenames: List[FilePath]) -> List[Detection]:
+        filepath_to_content_map: dict[FilePath, SecurityContentObject] = {
+        obj.file_path: obj for (_, obj) in self.director.name_to_content_map.items() if obj.file_path is not None
+    }
         errors = []
-        detections:List[Detection] = []
+        detections: List[Detection] = []
         for name in detectionFilenames:
-            obj = filepath_to_content_map.get(name,None)
-            if obj == None:
+            obj = filepath_to_content_map.get(name, None)
+            if obj is None:
                 errors.append(f"There is no detection file or security_content_object at '{name}'")
             elif not isinstance(obj, Detection):
                 errors.append(f"The security_content_object at '{name}' is of type '{type(obj).__name__}', NOT '{Detection.__name__}'")
             else:
                 detections.append(obj)
 
-        if len(errors) > 0:
+        if errors:
             errorsString = "\n - ".join(errors)
-            raise Exception(f"There following errors were encountered while getting selected detections to test:\n - {errorsString}")
-        return detections
-
+            raise Exception(f"The following errors were encountered while getting selected detections to test:\n - {errorsString}")
+        return detections