Merge pull request #203 from splunk/more_data_source_updates

The failures against ESCU are expected because data_source changes have not yet been made in the develop branch of that repo. As such, we accept those failures and merge anyway.

Author: pyth0n1c

contentctl/actions/build.py

@@ -11,6 +11,7 @@
 from contentctl.output.ba_yml_output import BAYmlOutput
 from contentctl.output.api_json_output import ApiJsonOutput
 from contentctl.output.data_source_writer import DataSourceWriter
+from contentctl.objects.lookup import Lookup
 import pathlib
 import json
 import datetime
@@ -34,7 +35,14 @@ def execute(self, input_dto: BuildInputDto) -> DirectorOutputDto:
             updated_conf_files:set[pathlib.Path] = set()
             conf_output = ConfOutput(input_dto.config)
 
-            DataSourceWriter.writeDataSourceCsv(input_dto.director_output_dto.data_sources, str(input_dto.config.path) + "/lookups/data_sources.csv")
+            # Construct a special lookup whose CSV is created at runtime and
+            # written directly into the output folder. It is created with model_construct,
+            # not model_validate, because the CSV does not exist yet.
+            data_sources_lookup_csv_path = input_dto.config.getPackageDirectoryPath() / "lookups" / "data_sources.csv"
+            DataSourceWriter.writeDataSourceCsv(input_dto.director_output_dto.data_sources, data_sources_lookup_csv_path)
+            input_dto.director_output_dto.addContentToDictMappings(Lookup.model_construct(description= "A lookup file that will contain the data source objects for detections.", 
+                                                                        filename=data_sources_lookup_csv_path, 
+                                                                        name="data_sources"))
 
             updated_conf_files.update(conf_output.writeHeaders())
             updated_conf_files.update(conf_output.writeObjects(input_dto.director_output_dto.detections, SecurityContentType.detections))

contentctl/actions/initialize.py

@@ -29,7 +29,6 @@ def execute(self, config: test) -> None:
             ('../templates/deployments/', 'deployments'),
             ('../templates/detections/', 'detections'),
             ('../templates/data_sources/', 'data_sources'),
-            ('../templates/event_sources/', 'event_sources'),
             ('../templates/macros/','macros'),
             ('../templates/stories/', 'stories'),
         ]:

contentctl/input/director.py

@@ -68,7 +68,7 @@ def addContentToDictMappings(self, content: SecurityContentObject):
             # for this function we prepend 'SSA ' to the name.
             content_name = f"SSA {content_name}"
 
-        if content_name in self.name_to_content_map and isinstance(self.name_to_content_map[content_name], type(content)):
+        if content_name in self.name_to_content_map:
             raise ValueError(
                 f"Duplicate name '{content_name}' with paths:\n"
                 f" - {content.file_path}\n"
@@ -131,6 +131,16 @@ def execute(self, input_dto: validate) -> None:
         self.createSecurityContent(SecurityContentType.detections)
         self.createSecurityContent(SecurityContentType.ssa_detections)
 
+        
+        from contentctl.objects.abstract_security_content_objects.detection_abstract import MISSING_SOURCES
+        if len(MISSING_SOURCES) > 0:
+            missing_sources_string = "\n 🟡 ".join(sorted(list(MISSING_SOURCES)))
+            print("WARNING: The following data_sources have been used in detections, but are not yet defined.\n"
+                  "This is not yet an error since not all data_sources have been defined, but will be convered to an error soon:\n 🟡 "
+                  f"{missing_sources_string}")
+        else:
+            print("No missing data_sources!")
+
     def createSecurityContent(self, contentType: SecurityContentType) -> None:
         if contentType == SecurityContentType.ssa_detections:
             files = Utils.get_all_yml_files_from_directory(

contentctl/input/yml_reader.py

@@ -40,10 +40,8 @@ def load_file(file_path: pathlib.Path, add_fields=True, STRICT_YML_CHECKING=Fals
         if add_fields == False:
             return yml_obj
 
-        try:
-            yml_obj['file_path'] = str(file_path)
-        except Exception as e:
-            import code
-            code.interact(local=locals())
+        
+        yml_obj['file_path'] = str(file_path)
+    
 
         return yml_obj

contentctl/objects/abstract_security_content_objects/detection_abstract.py

@@ -29,6 +29,7 @@
 from contentctl.objects.enums import ProvidingTechnology
 from contentctl.enrichments.cve_enrichment import CveEnrichmentObj
 
+MISSING_SOURCES:set[str] = set()
 
 class Detection_Abstract(SecurityContentObject):
     model_config = ConfigDict(use_enum_values=True)
@@ -402,12 +403,16 @@ def model_post_init(self, ctx:dict[str,Any]):
         sources = sorted(list(updated_data_source_names))
 
         matched_data_sources:list[DataSource] = []
-        missing_sources: list[str] = []
+        missing_sources:list[str] = []
         for source in sources:
             try:
                 matched_data_sources += DataSource.mapNamesToSecurityContentObjects([source], director)
             except Exception as data_source_mapping_exception:
-                missing_sources.append(source)
+                # We gobble this up and add it to a global set so that we
+                # can print it ONCE at the end of the build of datasources.
+                # This will be removed later as per the note below
+                MISSING_SOURCES.add(source)
+                
         if len(missing_sources) > 0:
             # This will be changed to ValueError when we have a complete list of data sources
             print(f"WARNING: The following exception occurred when mapping the data_source field to DataSource objects:{missing_sources}")

contentctl/objects/abstract_security_content_objects/security_content_object_abstract.py

@@ -125,9 +125,9 @@ def mapNamesToSecurityContentObjects(cls, v: list[str], director:Union[DirectorO
         errors:list[str] = []
         if len(missing_objects) > 0:
             errors.append(f"Failed to find the following '{cls.__name__}': {missing_objects}")
-        if len(missing_objects) > 0:
+        if len(mistyped_objects) > 0:
             for mistyped_object in mistyped_objects:
-                errors.append(f"'{mistyped_object.name}' expected to have type '{type(Self)}', but actually had type '{type(mistyped_object)}'")
+                errors.append(f"'{mistyped_object.name}' expected to have type '{cls}', but actually had type '{type(mistyped_object)}'")
 
         if len(errors) > 0:
             error_string = "\n  - ".join(errors)

contentctl/templates/data_sources/Sysmon_EventID.yml

@@ -1,7 +1,16 @@
 name: Sysmon EventID 1
 id: b375f4d1-d7ca-4bc0-9103-294825c0af17
+version: 1
+date: '2024-07-18'
 author: Patrick Bareiss, Splunk
-description: Event source object for Sysmon EventID 1
+description: Data source object for Sysmon EventID 1
+source: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational
+sourcetype: xmlwineventlog
+separator: EventID
+supported_TA:
+- name: Splunk Add-on for Sysmon
+  url: https://splunkbase.splunk.com/app/5709/
+  version: 4.0.0
 fields:
 - _time
 - Channel
@@ -100,6 +109,46 @@ fields:
 - user
 - user_id
 - vendor_product
+field_mappings:
+  - data_model: cim
+    data_set: Endpoint.Processes
+    mapping:
+      ProcessGuid: Processes.process_guid
+      ProcessId: Processes.process_id
+      Image: Processes.process_path
+      Image|endswith: Processes.process_name
+      CommandLine: Processes.process
+      CurrentDirectory: Processes.process_current_directory
+      User: Processes.user
+      IntegrityLevel: Processes.process_integrity_level
+      Hashes: Processes.process_hash
+      ParentProcessGuid: Processes.parent_process_guid
+      ParentProcessId: Processes.parent_process_id
+      ParentImage: Processes.parent_process_name
+      ParentCommandLine: Processes.parent_process
+      Computer: Processes.dest
+      OriginalFileName: Processes.original_file_name
+convert_to_log_source:
+  - data_source: Windows Event Log Security 4688
+    mapping:
+      ProcessId: NewProcessId
+      Image: NewProcessName
+      Image|endswith: NewProcessName|endswith
+      CommandLine: Process_Command_Line
+      User: SubjectUserSid
+      ParentProcessId: ProcessId
+      ParentImage: ParentProcessName
+      ParentImage|endswith: ParentProcessName|endswith
+      Computer: Computer
+      OriginalFileName: NewProcessName|endswith
+  - data_source: Crowdstrike Process
+    mapping:
+      ProcessId: RawProcessId
+      Image: ImageFileName
+      CommandLine: CommandLine
+      User: UserSid
+      ParentProcessId: ParentProcessId
+      ParentImage: ParentBaseFileName
 example_log: "<Event xmlns='http://schemas.microsoft.com/win/2004/08/events/event'><System><Provider\
   \ Name='Microsoft-Windows-Sysmon' Guid='{5770385F-C22A-43E0-BF4C-06F5698FFBD9}'/><EventID>1</EventID><Version>5</Version><Level>4</Level><Task>1</Task><Opcode>0</Opcode><Keywords>0x8000000000000000</Keywords><TimeCreated\
   \ SystemTime='2020-10-08T11:03:46.617920300Z'/><EventRecordID>4522</EventRecordID><Correlation/><Execution\