Merge pull request #90 from splunk/pydantic2

We're happy to announce the merge for contentctl 4.0!
![image](https://github.com/splunk/contentctl/assets/87383215/5152e9d2-3ce9-4570-b429-150dcadd3bcc)

Author: pyth0n1c

.github/workflows/testEndToEnd.yml

@@ -11,22 +11,22 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python_version: ["3.9", "3.10", "3.11"]
-        operating_system: ["ubuntu-20.04", "ubuntu-22.04"]
+        python_version: ["3.11", "3.12"]
+        operating_system: ["ubuntu-20.04", "ubuntu-22.04", "macos-latest", "macos-14"]
         #operating_system: ["ubuntu-20.04", "ubuntu-22.04", "macos-latest"]
 
 
     runs-on: ${{ matrix.operating_system }}
     steps:
-      - name: Install Docker for macOS
-        run: |
-          brew install docker
-          #import magic fails on macos runner
-          brew install libmagic
-          colima start
-          # Mapping below is required to get the Python docker library working
-          sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock
-        if: matrix.operating_system == 'macos-latest'
+      #- name: Install Docker for macOS
+      #  run: |
+      #     brew install docker
+      #     # import magic fails on macos runner
+      #     brew install libmagic colima
+      #     colima start
+      #     # Mapping below is required to get the Python docker library working
+      #     sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock
+      #  if: matrix.operating_system == 'macos-latest'
 
       #Checkout the current branch
       - name: Checkout repo
@@ -51,7 +51,12 @@ jobs:
       - name: Run contentctl init
         run: |
           cd my_splunk_content_pack
-          poetry run contentctl init
+          poetry run contentctl init  
+
+      - name: Clone the AtomicRedTeam Repo
+        run: |
+          cd my_splunk_content_pack
+          git clone --depth 1 https://github.com/redcanaryco/atomic-red-team
 
       - name: Run contentctl validate
         run: |
@@ -65,9 +70,10 @@ jobs:
 
       #Do not pause on a failed detection
       - name: Run contentctl test
+        if: startsWith(matrix.operating_system, 'ubuntu')
         run: |
           cd my_splunk_content_pack
-          poetry run contentctl test --unattended
+          poetry run contentctl test --disable-tqdm --post-test-behavior never_pause
       
       - uses: actions/upload-artifact@v4
         with:

.vscode/launch.json

@@ -1,19 +1,68 @@
 {
-    // Use IntelliSense to learn about possible attributes.
-    // Hover to view descriptions of existing attributes.
-    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
-    "version": "0.2.0",
-    "configurations": [
-
-        {
-            "name": "contentctl test",
-            "type": "python",
-            "request": "launch",
-            "program": "${workspaceFolder}/splunk_contentctl/contentctl.py",
-            "cwd": "${workspaceFolder}/splunk_contentctl",
-            "console": "integratedTerminal",
-            "justMyCode": true,
-            "args": ["-p", "tmp", "test"]
-        }
-    ]
+"configurations": [
+    {
+        "name": "contentctl init",
+        "type": "debugpy",
+        "request": "launch",
+        "program": "${workspaceFolder}/.venv/bin/contentctl",
+        "cwd": "${workspaceFolder}/../ddd/",
+        "args": ["init"]
+    },
+    {
+        "name": "contentctl validate",
+        "type": "debugpy",
+        "request": "launch",
+        "program": "${workspaceFolder}/.venv/bin/contentctl",
+        "cwd": "${workspaceFolder}/../",
+        "args": ["validate"]
+    },
+    {
+        "name": "contentctl validate enrich",
+        "type": "debugpy",
+        "request": "launch",
+        "program": "${workspaceFolder}/.venv/bin/contentctl",
+        "cwd": "${workspaceFolder}/../",
+        "args": ["validate", "--enrichments"]
+    },
+    {
+        "name": "contentctl build",
+        "type": "debugpy",
+        "request": "launch",
+        "program": "${workspaceFolder}/.venv/bin/contentctl",
+        "cwd": "${workspaceFolder}/../",
+        "args": ["build"]
+    },
+    {
+        "name": "contentctl build enrich",
+        "type": "debugpy",
+        "request": "launch",
+        "program": "${workspaceFolder}/.venv/bin/contentctl",
+        "cwd": "${workspaceFolder}/../",
+        "args": ["build", "--enrichments"]
+    },
+    {
+        "name": "contentctl test",
+        "type": "debugpy",
+        "request": "launch",
+        "program": "${workspaceFolder}/.venv/bin/contentctl",
+        "cwd": "${workspaceFolder}/../",
+        "args": ["test"]
+    },
+    {
+        "name": "contentctl --help",
+        "type": "debugpy",
+        "request": "launch",
+        "program": "${workspaceFolder}/.venv/bin/contentctl",
+        "cwd": "${workspaceFolder}/../",
+        "args": ["--help"]
+    },
+    {
+        "name": "contentctl test detection",
+        "type": "debugpy",
+        "request": "launch",
+        "program": "${workspaceFolder}/.venv/bin/contentctl",
+        "cwd": "${workspaceFolder}/../",
+        "args": ["test", "mode:selected", "--mode.files", "detections/endpoint/3cx_supply_chain_attack_network_indicators.yml"]
+    }
+]
 }

.vscode/settings.json

@@ -1,5 +1,9 @@
 {
     "python.terminal.activateEnvironment": true,
     "python.envFile": "${workspaceFolder}/.env",
-    "python.testing.cwd": "${workspaceFolder}"
+    "python.testing.cwd": "${workspaceFolder}",
+    "python.languageServer": "Pylance",
+    "python.analysis.typeCheckingMode": "strict"
+
+
 }

README.md

@@ -178,7 +178,28 @@ contentctl test's default mode allows it to quickly test all content with requir
 6. **docs** - Create documentation as Markdown
 7. **reporting** - Create different reporting files such as a Mitre ATT&CK overlay
 
+# Shell tab-complete
 
+Leveraging the tab completion featureset of the CLI library we're using, you can generate tab completions for `contentctl` automatically, for zsh, bash, and tcsh. For additional details, you can view the docs for the library [here.](https://brentyi.github.io/tyro/tab_completion/) 
+
+### Zsh
+If you already have a location for your ZSH tab completions, you only need to run the generation line and can skip the folder creation, configuring the rest to fit with your shell config.
+
+```zsh
+mkdir -p ~/.zfunc
+contentctl --tyro-write-completion zsh ~/.zfunc/_contentctl
+echo "fpath+=~/.zfunc" >> ~/.zshrc
+echo "autoload -Uz compinit && compinit" >> ~/.zshrc
+source ~/.zshrc
+```
+
+### Bash
+
+```bash
+completion_dir=${BASH_COMPLETION_USER_DIR:-${XDG_DATA_HOME:-$HOME/.local/share}/bash-completion}/completions/
+mkdir -p $completion_dir
+contentctl --tyro-write-completion bash ${completion_dir}/_contentctl
+```
 
 # Acronyms
 | Acronym | Meaning| Description | 

contentctl/actions/build.py

@@ -0,0 +1,89 @@
+import sys
+import shutil
+import os
+
+from dataclasses import dataclass
+
+from contentctl.objects.enums import SecurityContentProduct, SecurityContentType
+from contentctl.input.director import Director, DirectorOutputDto
+from contentctl.output.conf_output import ConfOutput
+from contentctl.output.conf_writer import ConfWriter
+from contentctl.output.ba_yml_output import BAYmlOutput
+from contentctl.output.api_json_output import ApiJsonOutput
+import pathlib
+import json
+import datetime
+from typing import Union
+
+from contentctl.objects.config import build
+
+@dataclass(frozen=True)
+class BuildInputDto:
+    director_output_dto: DirectorOutputDto
+    config:build
+
+
+class Build:
+
+
+
+    def execute(self, input_dto: BuildInputDto) -> DirectorOutputDto:
+        if input_dto.config.build_app:    
+            updated_conf_files:set[pathlib.Path] = set()
+            conf_output = ConfOutput(input_dto.config)
+            updated_conf_files.update(conf_output.writeHeaders())
+            updated_conf_files.update(conf_output.writeObjects(input_dto.director_output_dto.detections, SecurityContentType.detections))
+            updated_conf_files.update(conf_output.writeObjects(input_dto.director_output_dto.stories, SecurityContentType.stories))
+            updated_conf_files.update(conf_output.writeObjects(input_dto.director_output_dto.baselines, SecurityContentType.baselines))
+            updated_conf_files.update(conf_output.writeObjects(input_dto.director_output_dto.investigations, SecurityContentType.investigations))
+            updated_conf_files.update(conf_output.writeObjects(input_dto.director_output_dto.lookups, SecurityContentType.lookups))
+            updated_conf_files.update(conf_output.writeObjects(input_dto.director_output_dto.macros, SecurityContentType.macros))
+            updated_conf_files.update(conf_output.writeAppConf())
+            
+            #Ensure that the conf file we just generated/update is syntactically valid
+            for conf_file in updated_conf_files:
+                ConfWriter.validateConfFile(conf_file) 
+                
+            conf_output.packageApp()
+
+            print(f"Build of '{input_dto.config.app.title}' APP successful to {input_dto.config.getPackageFilePath()}")
+        
+
+        if input_dto.config.build_api:    
+            shutil.rmtree(input_dto.config.getAPIPath(), ignore_errors=True)
+            input_dto.config.getAPIPath().mkdir(parents=True)
+            api_json_output = ApiJsonOutput()
+            for output_objects, output_type in [(input_dto.director_output_dto.detections, SecurityContentType.detections),
+                                                (input_dto.director_output_dto.stories, SecurityContentType.stories),
+                                                (input_dto.director_output_dto.baselines, SecurityContentType.baselines),
+                                                (input_dto.director_output_dto.investigations, SecurityContentType.investigations),
+                                                (input_dto.director_output_dto.lookups, SecurityContentType.lookups),
+                                                (input_dto.director_output_dto.macros, SecurityContentType.macros),
+                                                (input_dto.director_output_dto.deployments, SecurityContentType.deployments)]:
+                api_json_output.writeObjects(output_objects, input_dto.config.getAPIPath(), input_dto.config.app.label, output_type )
+            
+           
+            
+            #create version file for sse api
+            version_file = input_dto.config.getAPIPath()/"version.json"
+            utc_time = datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0,tzinfo=None).isoformat()
+            version_dict = {"version":{"name":f"v{input_dto.config.app.version}","published_at": f"{utc_time}Z"  }}
+            with open(version_file,"w") as version_f:
+                json.dump(version_dict,version_f)
+            
+            print(f"Build of '{input_dto.config.app.title}' API successful to {input_dto.config.getAPIPath()}")
+
+        if input_dto.config.build_ssa:
+            
+            srs_path = input_dto.config.getSSAPath() / 'srs'
+            complex_path = input_dto.config.getSSAPath() / 'complex'
+            shutil.rmtree(srs_path, ignore_errors=True)
+            shutil.rmtree(complex_path, ignore_errors=True)
+            srs_path.mkdir(parents=True)
+            complex_path.mkdir(parents=True)
+            ba_yml_output = BAYmlOutput()
+            ba_yml_output.writeObjects(input_dto.director_output_dto.ssa_detections, str(input_dto.config.getSSAPath()))
+
+            print(f"Build of 'SSA' successful to {input_dto.config.getSSAPath()}")
+                
+        return input_dto.director_output_dto