Merge branch 'main' into release_notes_fix

Author: patel-bhavin

.github/ISSUE_TEMPLATE/bug_report.md

@@ -4,6 +4,7 @@ about: Create a report to help us improve contentctl
 title: "[BUG]"
 labels: bug
 assignees: ''
+type: "Bug"
 
 ---
 

.github/ISSUE_TEMPLATE/feature_request.md

@@ -4,7 +4,7 @@ about: Suggest an idea for this project
 title: ''
 labels: enhancement
 assignees: ''
-
+type: "Feature"
 ---
 
 **Is your feature request related to a problem? Please describe.**

.pre-commit-config.yaml

@@ -8,7 +8,7 @@ repos:
       - id: detect-private-key
       - id: forbid-submodules
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.9.10
+    rev: v0.11.2
     hooks:
       - id: ruff
         args: [ --fix ]

contentctl/actions/build.py

@@ -1,19 +1,14 @@
+import datetime
+import json
+import pathlib
 import shutil
-
 from dataclasses import dataclass
 
 from contentctl.input.director import DirectorOutputDto
+from contentctl.objects.config import build
+from contentctl.output.api_json_output import JSON_API_VERSION, ApiJsonOutput
 from contentctl.output.conf_output import ConfOutput
 from contentctl.output.conf_writer import ConfWriter
-from contentctl.output.api_json_output import ApiJsonOutput
-from contentctl.output.data_source_writer import DataSourceWriter
-from contentctl.objects.lookup import CSVLookup, Lookup_Type
-import pathlib
-import json
-import datetime
-import uuid
-
-from contentctl.objects.config import build
 
 
 @dataclass(frozen=True)
@@ -28,39 +23,6 @@ def execute(self, input_dto: BuildInputDto) -> DirectorOutputDto:
             updated_conf_files: set[pathlib.Path] = set()
             conf_output = ConfOutput(input_dto.config)
 
-            # Construct a path to a YML that does not actually exist.
-            # We mock this "fake" path since the YML does not exist.
-            # This ensures the checking for the existence of the CSV is correct
-            data_sources_fake_yml_path = (
-                input_dto.config.getPackageDirectoryPath()
-                / "lookups"
-                / "data_sources.yml"
-            )
-
-            # Construct a special lookup whose CSV is created at runtime and
-            # written directly into the lookups folder. We will delete this after a build,
-            # assuming that it is successful.
-            data_sources_lookup_csv_path = (
-                input_dto.config.getPackageDirectoryPath()
-                / "lookups"
-                / "data_sources.csv"
-            )
-
-            DataSourceWriter.writeDataSourceCsv(
-                input_dto.director_output_dto.data_sources, data_sources_lookup_csv_path
-            )
-            input_dto.director_output_dto.addContentToDictMappings(
-                CSVLookup.model_construct(
-                    name="data_sources",
-                    id=uuid.UUID("b45c1403-6e09-47b0-824f-cf6e44f15ac8"),
-                    version=1,
-                    author=input_dto.config.app.author_name,
-                    date=datetime.date.today(),
-                    description="A lookup file that will contain the data source objects for detections.",
-                    lookup_type=Lookup_Type.csv,
-                    file_path=data_sources_fake_yml_path,
-                )
-            )
             updated_conf_files.update(conf_output.writeHeaders())
             updated_conf_files.update(
                 conf_output.writeLookups(input_dto.director_output_dto.lookups)
@@ -114,7 +76,9 @@ def execute(self, input_dto: BuildInputDto) -> DirectorOutputDto:
             api_json_output.writeDeployments(input_dto.director_output_dto.deployments)
 
             # create version file for sse api
-            version_file = input_dto.config.getAPIPath() / "version.json"
+            version_file = (
+                input_dto.config.getAPIPath() / f"version_v{JSON_API_VERSION}.json"
+            )
             utc_time = (
                 datetime.datetime.now(datetime.timezone.utc)
                 .replace(microsecond=0, tzinfo=None)

contentctl/actions/detection_testing/DetectionTestingManager.py

@@ -1,32 +1,29 @@
+import concurrent.futures
+import datetime
+import signal
+import traceback
+from dataclasses import dataclass
 from typing import List, Union
-from contentctl.objects.config import test, test_servers, Container, Infrastructure
+
+import docker
+from pydantic import BaseModel
+
 from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructure import (
     DetectionTestingInfrastructure,
+    DetectionTestingManagerOutputDto,
 )
 from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructureContainer import (
     DetectionTestingInfrastructureContainer,
 )
 from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructureServer import (
     DetectionTestingInfrastructureServer,
 )
-import signal
-import datetime
-
-# from queue import Queue
-from dataclasses import dataclass
-
-# import threading
-from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructure import (
-    DetectionTestingManagerOutputDto,
-)
 from contentctl.actions.detection_testing.views.DetectionTestingView import (
     DetectionTestingView,
 )
-from contentctl.objects.enums import PostTestBehavior
-from pydantic import BaseModel
+from contentctl.objects.config import Container, Infrastructure, test, test_servers
 from contentctl.objects.detection import Detection
-import concurrent.futures
-import docker
+from contentctl.objects.enums import PostTestBehavior
 
 
 @dataclass(frozen=False)
@@ -63,12 +60,14 @@ def sigint_handler(signum, frame):
                 # a newline '\r\n' which will cause that wait to stop
                 print("*******************************")
                 print(
-                    "If testing is paused and you are debugging a detection, you MUST hit CTRL-D at the prompt to complete shutdown."
+                    "If testing is paused and you are debugging a detection, you MUST hit CTRL-D "
+                    "at the prompt to complete shutdown."
                 )
                 print("*******************************")
 
         signal.signal(signal.SIGINT, sigint_handler)
 
+        # TODO (#337): futures can be hard to maintain/debug; let's consider alternatives
         with (
             concurrent.futures.ThreadPoolExecutor(
                 max_workers=len(self.input_dto.config.test_instances),
@@ -80,10 +79,19 @@ def sigint_handler(signum, frame):
                 max_workers=len(self.input_dto.config.test_instances),
             ) as view_shutdowner,
         ):
+            # Capture any errors for reporting at the end after all threads have been gathered
+            errors: dict[str, list[Exception]] = {
+                "INSTANCE SETUP ERRORS": [],
+                "TESTING ERRORS": [],
+                "ERRORS DURING VIEW SHUTDOWN": [],
+                "ERRORS DURING VIEW EXECUTION": [],
+            }
+
             # Start all the views
             future_views = {
                 view_runner.submit(view.setup): view for view in self.input_dto.views
             }
+
             # Configure all the instances
             future_instances_setup = {
                 instance_pool.submit(instance.setup): instance
@@ -96,7 +104,11 @@ def sigint_handler(signum, frame):
                     future.result()
                 except Exception as e:
                     self.output_dto.terminate = True
-                    print(f"Error setting up container: {str(e)}")
+                    # Output the traceback if we encounter errors in verbose mode
+                    if self.input_dto.config.verbose:
+                        tb = traceback.format_exc()
+                        print(tb)
+                    errors["INSTANCE SETUP ERRORS"].append(e)
 
             # Start and wait for all tests to run
             if not self.output_dto.terminate:
@@ -111,7 +123,11 @@ def sigint_handler(signum, frame):
                         future.result()
                     except Exception as e:
                         self.output_dto.terminate = True
-                        print(f"Error running in container: {str(e)}")
+                        # Output the traceback if we encounter errors in verbose mode
+                        if self.input_dto.config.verbose:
+                            tb = traceback.format_exc()
+                            print(tb)
+                        errors["TESTING ERRORS"].append(e)
 
             self.output_dto.terminate = True
 
@@ -123,14 +139,34 @@ def sigint_handler(signum, frame):
                 try:
                     future.result()
                 except Exception as e:
-                    print(f"Error stopping view: {str(e)}")
+                    # Output the traceback if we encounter errors in verbose mode
+                    if self.input_dto.config.verbose:
+                        tb = traceback.format_exc()
+                        print(tb)
+                    errors["ERRORS DURING VIEW SHUTDOWN"].append(e)
 
             # Wait for original view-related threads to complete
             for future in concurrent.futures.as_completed(future_views):
                 try:
                     future.result()
                 except Exception as e:
-                    print(f"Error running container: {str(e)}")
+                    # Output the traceback if we encounter errors in verbose mode
+                    if self.input_dto.config.verbose:
+                        tb = traceback.format_exc()
+                        print(tb)
+                    errors["ERRORS DURING VIEW EXECUTION"].append(e)
+
+            # Log any errors
+            for error_type in errors:
+                if len(errors[error_type]) > 0:
+                    print()
+                    print(f"[{error_type}]:")
+                    for error in errors[error_type]:
+                        print(f"\t❌ {str(error)}")
+                        if isinstance(error, ExceptionGroup):
+                            for suberror in error.exceptions:  # type: ignore
+                                print(f"\t\t❌ {str(suberror)}")  # type: ignore
+                    print()
 
         return self.output_dto
 
@@ -154,12 +190,15 @@ def create_DetectionTestingInfrastructureObjects(self):
                     )
                     if len(parts) != 2:
                         raise Exception(
-                            f"Expected to find a name:tag in {self.input_dto.config.container_settings.full_image_path}, "
-                            f"but instead found {parts}. Note that this path MUST include the tag, which is separated by ':'"
+                            "Expected to find a name:tag in "
+                            f"{self.input_dto.config.container_settings.full_image_path}, "
+                            f"but instead found {parts}. Note that this path MUST include the "
+                            "tag, which is separated by ':'"
                         )
 
                     print(
-                        f"Getting the latest version of the container image [{self.input_dto.config.container_settings.full_image_path}]...",
+                        "Getting the latest version of the container image "
+                        f"[{self.input_dto.config.container_settings.full_image_path}]...",
                         end="",
                         flush=True,
                     )
@@ -168,7 +207,8 @@ def create_DetectionTestingInfrastructureObjects(self):
                     break
                 except Exception as e:
                     raise Exception(
-                        f"Failed to pull docker container image [{self.input_dto.config.container_settings.full_image_path}]: {str(e)}"
+                        "Failed to pull docker container image "
+                        f"[{self.input_dto.config.container_settings.full_image_path}]: {str(e)}"
                     )
 
         already_staged_container_files = False

contentctl/actions/detection_testing/GitService.py

@@ -14,7 +14,7 @@
 from contentctl.objects.config import All, Changes, Selected, test_common
 from contentctl.objects.data_source import DataSource
 from contentctl.objects.detection import Detection
-from contentctl.objects.lookup import CSVLookup, Lookup
+from contentctl.objects.lookup import CSVLookup, Lookup, RuntimeCSV
 from contentctl.objects.macro import Macro
 from contentctl.objects.security_content_object import SecurityContentObject
 
@@ -148,6 +148,9 @@ def getChanges(self, target_branch: str) -> List[Detection]:
                             matched = list(
                                 filter(
                                     lambda x: isinstance(x, CSVLookup)
+                                    and not isinstance(
+                                        x, RuntimeCSV
+                                    )  # RuntimeCSV is not used directly by any content
                                     and x.filename == decoded_path,
                                     self.director.lookups,
                                 )