Merge pull request #138 from splunk/pydantic2_bonus_content

Additional Pydantic2 content

Author: patel-bhavin

.github/workflows/testEndToEnd.yml

@@ -51,7 +51,7 @@ jobs:
       - name: Run contentctl init
         run: |
           cd my_splunk_content_pack
-          poetry run contentctl init
+          poetry run contentctl init  
 
       - name: Clone the AtomicRedTeam Repo
         run: |
@@ -73,7 +73,7 @@ jobs:
         if: startsWith(matrix.operating_system, 'ubuntu')
         run: |
           cd my_splunk_content_pack
-          poetry run contentctl test
+          poetry run contentctl test --disable-tqdm --post-test-behavior never_pause
       
       - uses: actions/upload-artifact@v4
         with:

contentctl/actions/detection_testing/DetectionTestingManager.py

@@ -154,18 +154,24 @@ def create_DetectionTestingInfrastructureObjects(self):
                 except Exception as e:
                     raise Exception(f"Failed to pull docker container image [{self.input_dto.config.container_settings.full_image_path}]: {str(e)}")
 
+        already_staged_container_files = False
         for infrastructure in self.input_dto.config.test_instances:
 
             if (isinstance(self.input_dto.config, test) and isinstance(infrastructure, Container)):
+                # Stage the files in the apps dir so that they can be passed directly to
+                # subsequent containers. Do this here, instead of inside each container, to
+                # avoid duplicate downloads/moves/copies
+                if not already_staged_container_files:
+                    self.input_dto.config.getContainerEnvironmentString(stage_file=True)
+                    already_staged_container_files = True
 
                 self.detectionTestingInfrastructureObjects.append(
                     DetectionTestingInfrastructureContainer(
                         global_config=self.input_dto.config, infrastructure=infrastructure, sync_obj=self.output_dto
                     )
                 )
 
-            elif isinstance(infrastructure, Infrastructure):
-
+            elif (isinstance(self.input_dto.config, test_servers) and isinstance(infrastructure, Infrastructure)):
                 self.detectionTestingInfrastructureObjects.append(
                     DetectionTestingInfrastructureServer(
                         global_config=self.input_dto.config, infrastructure=infrastructure, sync_obj=self.output_dto

contentctl/actions/detection_testing/GitService.py

@@ -14,7 +14,7 @@
 from contentctl.objects.lookup import Lookup
 from contentctl.objects.detection import Detection
 from contentctl.objects.security_content_object import SecurityContentObject
-from contentctl.objects.config import test, All, Changes, Selected
+from contentctl.objects.config import test_common, All, Changes, Selected
 
 # Logger
 logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO"))
@@ -28,7 +28,7 @@
 
 class GitService(BaseModel):
     director: DirectorOutputDto
-    config: test
+    config: test_common
     gitHash: Optional[str] = None
 
     def getHash(self)->str:

contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py

@@ -23,7 +23,7 @@
 from urllib3 import disable_warnings
 import urllib.parse
 
-from contentctl.objects.config import test, test_servers, test_common, Infrastructure
+from contentctl.objects.config import test_common, Infrastructure
 from contentctl.objects.enums import PostTestBehavior, AnalyticsType
 from contentctl.objects.detection import Detection
 from contentctl.objects.base_test import BaseTest
@@ -32,7 +32,6 @@
 from contentctl.objects.unit_test_attack_data import UnitTestAttackData
 from contentctl.objects.unit_test_result import UnitTestResult
 from contentctl.objects.integration_test_result import IntegrationTestResult
-#from contentctl.objects.test_config import TestConfig, Infrastructure
 from contentctl.objects.test_group import TestGroup
 from contentctl.objects.base_test_result import TestResultStatus
 from contentctl.objects.correlation_search import CorrelationSearch, PbarData
@@ -79,7 +78,7 @@ class DetectionTestingManagerOutputDto():
 
 class DetectionTestingInfrastructure(BaseModel, abc.ABC):
     # thread: threading.Thread = threading.Thread()
-    global_config: Union[test,test_servers]
+    global_config: test_common
     infrastructure: Infrastructure
     sync_obj: DetectionTestingManagerOutputDto
     hec_token: str = ""
@@ -396,7 +395,7 @@ def execute(self):
             try:
                 self.test_detection(detection)
             except ContainerStoppedException:
-                self.pbar.write(f"Stopped container [{self.get_name()}]")
+                self.pbar.write(f"Warning - container was stopped when trying to execute detection [{self.get_name()}]")
                 self.finish()
                 return
             except Exception as e:
@@ -1357,7 +1356,7 @@ def status(self):
         pass
 
     def finish(self):
-        self.pbar.bar_format = f"Stopped container [{self.get_name()}]"
+        self.pbar.bar_format = f"Finished running tests on instance: [{self.get_name()}]"
         self.pbar.update()
         self.pbar.close()
 

contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructureContainer.py

@@ -6,10 +6,6 @@
 import docker.models.containers
 import docker
 import docker.types
-from contentctl.objects.test_config import (
-    CONTAINER_APP_DIR,
-    LOCAL_APP_DIR,
-)
 
 
 class DetectionTestingInfrastructureContainer(DetectionTestingInfrastructure):
@@ -78,8 +74,8 @@ def make_container(self) -> docker.models.resource.Model:
 
         mounts = [
             docker.types.Mount(
-                source=str(LOCAL_APP_DIR.absolute()),
-                target=str(CONTAINER_APP_DIR.absolute()),
+                source=str(self.global_config.getLocalAppDir()),
+                target=str(self.global_config.getContainerAppDir()),
                 type="bind",
                 read_only=True,
             )
@@ -88,7 +84,9 @@ def make_container(self) -> docker.models.resource.Model:
         environment = {}
         environment["SPLUNK_START_ARGS"] = "--accept-license"
         environment["SPLUNK_PASSWORD"] = self.infrastructure.splunk_app_password
-        environment["SPLUNK_APPS_URL"] = self.global_config.getContainerEnvironmentString(stage_file=True)
+        # Files have already been staged by the time that we call this. Files must only be staged
+        # once, not staged by every container
+        environment["SPLUNK_APPS_URL"] = self.global_config.getContainerEnvironmentString(stage_file=False)
         if (
             self.global_config.splunk_api_username is not None
             and self.global_config.splunk_api_password is not None
@@ -119,6 +117,18 @@ def emit_docker_run_equivalent():
             detach=True,
             platform="linux/amd64"
         )
+        
+        if self.global_config.enterpriseSecurityInApps():
+            #ES sets up https, so make sure it is included in the link
+            address = f"https://{self.infrastructure.instance_address}:{self.infrastructure.web_ui_port}"
+        else:
+            address = f"http://{self.infrastructure.instance_address}:{self.infrastructure.web_ui_port}"
+        print(f"\nStarted container with the following information:\n"
+              f"\tname    : [{self.get_name()}]\n"
+              f"\taddress : [{address}]\n"
+              f"\tusername: [{self.infrastructure.splunk_app_username}]\n"
+              f"\tpassword: [{self.infrastructure.splunk_app_password}]\n"
+              )
 
         return container
 
@@ -140,6 +150,8 @@ def removeContainer(self, removeVolumes: bool = True, forceRemove: bool = True):
             # container was found, so now we try to remove it
             # v also removes volumes linked to the container
             container.remove(v=removeVolumes, force=forceRemove)
+            print(f"container [{self.get_name()}] successfully removed")
+
             # remove it even if it is running. remove volumes as well
             # No need to print that the container has been removed, it is expected behavior
 

contentctl/actions/detection_testing/views/DetectionTestingView.py

@@ -3,8 +3,7 @@
 
 from pydantic import BaseModel
 
-from contentctl.objects.test_config import TestConfig
-from contentctl.objects.config import test
+from contentctl.objects.config import test_common
 
 from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructure import (
     DetectionTestingManagerOutputDto,
@@ -14,7 +13,7 @@
 
 
 class DetectionTestingView(BaseModel, abc.ABC):
-    config: test
+    config: test_common
     sync_obj: DetectionTestingManagerOutputDto
 
     interval: float = 10

contentctl/actions/detection_testing/views/DetectionTestingViewFile.py

@@ -1,11 +1,3 @@
-from pydantic import BaseModel
-import abc
-from typing import Callable
-from contentctl.objects.test_config import TestConfig
-from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructure import (
-    DetectionTestingManagerOutputDto,
-)
-
 from contentctl.actions.detection_testing.views.DetectionTestingView import (
     DetectionTestingView,
 )

contentctl/actions/initialize.py

@@ -34,7 +34,7 @@ def execute(self, config: test) -> None:
             source_directory = pathlib.Path(os.path.dirname(__file__))/templateDir
             target_directory = config.path/targetDir
             #Throw an exception if the target exists
-            shutil.copytree(source_directory, targetDir, dirs_exist_ok=False)
+            shutil.copytree(source_directory, target_directory, dirs_exist_ok=False)
 
         #Create the config file as well
         shutil.copyfile(pathlib.Path(os.path.dirname(__file__))/'../templates/README','README')