Merge branch 'main' into snapattack_datasource_enrichments

Author: ljstella

.DS_Store

@@ -1,46 +1,46 @@
-import time
-import uuid
 import abc
-import os.path
 import configparser
-import json
 import datetime
-import tqdm  # type: ignore
+import json
+import os.path
 import pathlib
-from tempfile import TemporaryDirectory, mktemp
+import time
+import urllib.parse
+import uuid
+from shutil import copyfile
 from ssl import SSLEOFError, SSLZeroReturnError
 from sys import stdout
-from shutil import copyfile
-from typing import Union, Optional
+from tempfile import TemporaryDirectory, mktemp
+from typing import Optional, Union
 
-from pydantic import ConfigDict, BaseModel, PrivateAttr, Field, dataclasses
 import requests  # type: ignore
 import splunklib.client as client  # type: ignore
+import splunklib.results
+import tqdm  # type: ignore
+from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, dataclasses
 from splunklib.binding import HTTPError  # type: ignore
 from splunklib.results import JSONResultsReader, Message  # type: ignore
-import splunklib.results
 from urllib3 import disable_warnings
-import urllib.parse
 
-from contentctl.objects.config import test_common, Infrastructure
-from contentctl.objects.enums import PostTestBehavior, AnalyticsType
-from contentctl.objects.detection import Detection
-from contentctl.objects.base_test import BaseTest
-from contentctl.objects.unit_test import UnitTest
-from contentctl.objects.integration_test import IntegrationTest
-from contentctl.objects.test_attack_data import TestAttackData
-from contentctl.objects.unit_test_result import UnitTestResult
-from contentctl.objects.integration_test_result import IntegrationTestResult
-from contentctl.objects.test_group import TestGroup
-from contentctl.objects.base_test_result import TestResultStatus
-from contentctl.objects.correlation_search import CorrelationSearch, PbarData
-from contentctl.helper.utils import Utils
 from contentctl.actions.detection_testing.progress_bar import (
-    format_pbar_string,
-    TestReportingType,
     FinalTestingStates,
     TestingStates,
+    TestReportingType,
+    format_pbar_string,
 )
+from contentctl.helper.utils import Utils
+from contentctl.objects.base_test import BaseTest
+from contentctl.objects.base_test_result import TestResultStatus
+from contentctl.objects.config import Infrastructure, test_common
+from contentctl.objects.correlation_search import CorrelationSearch, PbarData
+from contentctl.objects.detection import Detection
+from contentctl.objects.enums import AnalyticsType, PostTestBehavior
+from contentctl.objects.integration_test import IntegrationTest
+from contentctl.objects.integration_test_result import IntegrationTestResult
+from contentctl.objects.test_attack_data import TestAttackData
+from contentctl.objects.test_group import TestGroup
+from contentctl.objects.unit_test import UnitTest
+from contentctl.objects.unit_test_result import UnitTestResult
 
 
 class SetupTestGroupResults(BaseModel):
@@ -1109,17 +1109,25 @@ def retry_search_until_timeout(
             job = self.get_conn().search(query=search, **kwargs)
             results = JSONResultsReader(job.results(output_mode="json"))
 
-            # TODO (cmcginley): @ljstella you're removing this ultimately, right?
-            # Consolidate a set of the distinct observable field names
-            observable_fields_set = set(
-                [o.name for o in detection.tags.observable]
-            )  # keeping this around for later
-            risk_object_fields_set = set(
-                [o.name for o in detection.tags.observable if "Victim" in o.role]
-            )  # just the "Risk Objects"
-            threat_object_fields_set = set(
-                [o.name for o in detection.tags.observable if "Attacker" in o.role]
-            )  # just the "threat objects"
+            if detection.rba is not None:
+                risk_object_fields_set = set(
+                    [o.field for o in detection.rba.risk_objects]
+                )  # just the "Risk Objects"
+                threat_object_fields_set = set(
+                    [o.field for o in detection.rba.threat_objects]
+                )  # just the "threat objects"
+            else:
+                # For some searches, like Hunting Searches, there should
+                # not be any risk or threat objects.
+                risk_object_fields_set: set[str] = (
+                    set()
+                )  # just the "Risk Objects" (of which there are none)
+                threat_object_fields_set: set[str] = (
+                    set()
+                )  # just the "threat objects" (of which there are none)
+            full_rba_field_set: set[str] = risk_object_fields_set.union(
+                threat_object_fields_set
+            )
 
             # Ensure the search had at least one result
             if int(job.content.get("resultCount", "0")) > 0:
@@ -1164,7 +1172,7 @@ def retry_search_until_timeout(
 
                     # TODO (cmcginley): @ljstella is this something we're keeping for testing as
                     #   well?
-                    for field in observable_fields_set:
+                    for field in full_rba_field_set:
                         if result.get(field, "null") == "null":
                             if field in risk_object_fields_set:
                                 e = Exception(

contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py

@@ -1,23 +1,45 @@
-import sys
-from dataclasses import dataclass
-import pathlib
-import json
 import datetime
-import timeit
+import json
+import pathlib
+import sys
 import time
+import timeit
+from dataclasses import dataclass
+from io import BufferedReader
 
-from requests import Session, post, get
+from requests import Session, get, post
 from requests.auth import HTTPBasicAuth
 
 from contentctl.objects.config import inspect
-from contentctl.objects.savedsearches_conf import SavedsearchesConf
 from contentctl.objects.errors import (
-    MetadataValidationError,
     DetectionIDError,
     DetectionMissingError,
-    VersionDecrementedError,
+    MetadataValidationError,
     VersionBumpingError,
+    VersionDecrementedError,
 )
+from contentctl.objects.savedsearches_conf import SavedsearchesConf
+
+"""
+The following list includes all appinspect tags available from:
+https://dev.splunk.com/enterprise/reference/appinspect/appinspecttagreference/
+
+This allows contentctl to be as forward-leaning as possible in catching
+any potential issues on the widest variety of stacks.
+"""
+INCLUDED_TAGS_LIST = [
+    "aarch64_compatibility",
+    "ast",
+    "cloud",
+    "future",
+    "manual",
+    "packaging_standards",
+    "private_app",
+    "private_classic",
+    "private_victoria",
+    "splunk_appinspect",
+]
+INCLUDED_TAGS_STRING = ",".join(INCLUDED_TAGS_LIST)
 
 
 @dataclass(frozen=True)
@@ -28,7 +50,6 @@ class InspectInputDto:
 class Inspect:
     def execute(self, config: inspect) -> str:
         if config.build_app or config.build_api:
-            self.inspectAppCLI(config)
             appinspect_token = self.inspectAppAPI(config)
 
             if config.enable_metadata_validation:
@@ -49,10 +70,6 @@ def inspectAppAPI(self, config: inspect) -> str:
         session.auth = HTTPBasicAuth(
             config.splunk_api_username, config.splunk_api_password
         )
-        if config.stack_type not in ["victoria", "classic"]:
-            raise Exception(
-                f"stack_type MUST be either 'classic' or 'victoria', NOT '{config.stack_type}'"
-            )
 
         APPINSPECT_API_LOGIN = "https://api.splunk.com/2.0/rest/login/splunk"
 
@@ -64,10 +81,6 @@ def inspectAppAPI(self, config: inspect) -> str:
         APPINSPECT_API_VALIDATION_REQUEST = (
             "https://appinspect.splunk.com/v1/app/validate"
         )
-        headers = {
-            "Authorization": f"bearer {authorization_bearer}",
-            "Cache-Control": "no-cache",
-        }
 
         package_path = config.getPackageFilePath(include_version=False)
         if not package_path.is_file():
@@ -77,18 +90,43 @@ def inspectAppAPI(self, config: inspect) -> str:
                 "trying to 'contentctl deploy_acs' the package BEFORE running 'contentctl build'?"
             )
 
-        files = {
+        """
+        Some documentation on "files" argument for requests.post exists here:
+        https://docs.python-requests.org/en/latest/api/
+        The type (None, INCLUDED_TAGS_STRING) is intentional, and the None is important.
+        In curl syntax, the request we make below is equivalent to
+        curl -X POST \
+            -H "Authorization: bearer <TOKEN>" \
+            -H "Cache-Control: no-cache" \
+            -F "app_package=@<PATH/APP-PACKAGE>" \
+            -F "included_tags=cloud" \
+            --url "https://appinspect.splunk.com/v1/app/validate"
+        
+        This is confirmed by the great resource:
+        https://curlconverter.com/
+        """
+        data: dict[str, tuple[None, str] | BufferedReader] = {
             "app_package": open(package_path, "rb"),
-            "included_tags": (None, "cloud"),
+            "included_tags": (
+                None,
+                INCLUDED_TAGS_STRING,
+            ),  # tuple with None is intentional here
         }
 
-        res = post(APPINSPECT_API_VALIDATION_REQUEST, headers=headers, files=files)
+        headers = {
+            "Authorization": f"bearer {authorization_bearer}",
+            "Cache-Control": "no-cache",
+        }
+
+        res = post(APPINSPECT_API_VALIDATION_REQUEST, files=data, headers=headers)
 
         res.raise_for_status()
 
         request_id = res.json().get("request_id", None)
-        APPINSPECT_API_VALIDATION_STATUS = f"https://appinspect.splunk.com/v1/app/validate/status/{request_id}?included_tags=private_{config.stack_type}"
-        headers = headers = {"Authorization": f"bearer {authorization_bearer}"}
+        APPINSPECT_API_VALIDATION_STATUS = (
+            f"https://appinspect.splunk.com/v1/app/validate/status/{request_id}"
+        )
+
         startTime = timeit.default_timer()
         # the first time, wait for 40 seconds. subsequent times, wait for less.
         # this is because appinspect takes some time to return, so there is no sense
@@ -114,7 +152,9 @@ def inspectAppAPI(self, config: inspect) -> str:
                 raise Exception(f"Error - Unknown Appinspect API status '{status}'")
 
         # We have finished running appinspect, so get the report
-        APPINSPECT_API_REPORT = f"https://appinspect.splunk.com/v1/app/report/{request_id}?included_tags=private_{config.stack_type}"
+        APPINSPECT_API_REPORT = (
+            f"https://appinspect.splunk.com/v1/app/report/{request_id}"
+        )
         # Get human-readable HTML report
         headers = headers = {
             "Authorization": f"bearer {authorization_bearer}",
@@ -159,14 +199,14 @@ def inspectAppCLI(self, config: inspect) -> None:
                 "\t - https://dev.splunk.com/enterprise/docs/developapps/testvalidate/appinspect/useappinspectclitool/"
             )
             from splunk_appinspect.main import (
-                validate,
-                MODE_OPTION,
                 APP_PACKAGE_ARGUMENT,
-                OUTPUT_FILE_OPTION,
-                LOG_FILE_OPTION,
-                INCLUDED_TAGS_OPTION,
                 EXCLUDED_TAGS_OPTION,
+                INCLUDED_TAGS_OPTION,
+                LOG_FILE_OPTION,
+                MODE_OPTION,
+                OUTPUT_FILE_OPTION,
                 TEST_MODE,
+                validate,
             )
         except Exception as e:
             print(e)

contentctl/actions/inspect.py

@@ -1,19 +1,17 @@
-import questionary
-from typing import Any
-from contentctl.input.new_content_questions import NewContentQuestions
-from contentctl.objects.config import new, NewContentType
+import pathlib
 import uuid
 from datetime import datetime
-import pathlib
+from typing import Any
+
+import questionary
+
+from contentctl.input.new_content_questions import NewContentQuestions
 from contentctl.objects.abstract_security_content_objects.security_content_object_abstract import (
     SecurityContentObject_Abstract,
 )
-from contentctl.output.yml_writer import YmlWriter
+from contentctl.objects.config import NewContentType, new
 from contentctl.objects.enums import AssetType
-from contentctl.objects.constants import (
-    SES_OBSERVABLE_TYPE_MAPPING,
-    SES_OBSERVABLE_ROLE_MAPPING,
-)
+from contentctl.output.yml_writer import YmlWriter
 
 
 class NewContent:
@@ -34,6 +32,14 @@ class NewContent:
         },
     ]
 
+    DEFAULT_RBA = {
+        "message": "Risk Message goes here",
+        "risk_objects": [{"field": "dest", "type": "system", "score": 10}],
+        "threat_objects": [
+            {"field": "parent_process_name", "type": "parent_process_name"}
+        ],
+    }
+
     def buildDetection(self) -> tuple[dict[str, Any], str]:
         questions = NewContentQuestions.get_questions_detection()
         answers: dict[str, str] = questionary.prompt(
@@ -44,8 +50,8 @@ def buildDetection(self) -> tuple[dict[str, Any], str]:
             raise ValueError("User didn't answer one or more questions!")
 
         data_source_field = (
-            answers["data_source"]
-            if len(answers["data_source"]) > 0
+            answers["data_sources"]
+            if len(answers["data_sources"]) > 0
             else [f"{NewContent.UPDATE_PREFIX} zero or more data_sources"]
         )
         file_name = (
@@ -85,24 +91,13 @@ def buildDetection(self) -> tuple[dict[str, Any], str]:
                 f"{NewContent.UPDATE_PREFIX} zero or more http references to provide more information about your search"
             ],
             "drilldown_searches": NewContent.DEFAULT_DRILLDOWN_DEF,
+            "rba": NewContent.DEFAULT_RBA,
             "tags": {
                 "analytic_story": [
                     f"{NewContent.UPDATE_PREFIX} by providing zero or more analytic stories"
                 ],
                 "asset_type": f"{NewContent.UPDATE_PREFIX} by providing and asset type from {list(AssetType._value2member_map_)}",
-                "confidence": f"{NewContent.UPDATE_PREFIX} by providing a value between 1-100",
-                "impact": f"{NewContent.UPDATE_PREFIX} by providing a value between 1-100",
-                "message": f"{NewContent.UPDATE_PREFIX} by providing a risk message. Fields in your search results can be referenced using $fieldName$",
                 "mitre_attack_id": mitre_attack_ids,
-                "observable": [
-                    {
-                        "name": f"{NewContent.UPDATE_PREFIX} the field name of the observable. This is a field that exists in your search results.",
-                        "type": f"{NewContent.UPDATE_PREFIX} the type of your observable from the list {list(SES_OBSERVABLE_TYPE_MAPPING.keys())}.",
-                        "role": [
-                            f"{NewContent.UPDATE_PREFIX} the role from the list {list(SES_OBSERVABLE_ROLE_MAPPING.keys())}"
-                        ],
-                    }
-                ],
                 "product": [
                     "Splunk Enterprise",
                     "Splunk Enterprise Security",
@@ -128,6 +123,9 @@ def buildDetection(self) -> tuple[dict[str, Any], str]:
         if answers["detection_type"] not in ["TTP", "Anomaly", "Correlation"]:
             del output_file_answers["drilldown_searches"]
 
+        if answers["detection_type"] not in ["TTP", "Anomaly"]:
+            del output_file_answers["rba"]
+
         return output_file_answers, answers["detection_kind"]
 
     def buildStory(self) -> dict[str, Any]:
@@ -142,6 +140,7 @@ def buildStory(self) -> dict[str, Any]:
         del answers["story_name"]
         answers["id"] = str(uuid.uuid4())
         answers["version"] = 1
+        answers["status"] = "production"
         answers["date"] = datetime.today().strftime("%Y-%m-%d")
         answers["author"] = answers["story_author"]
         del answers["story_author"]